2024-12-03 14:47:57,602 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-03 14:47:57,623 main DEBUG Took 0.018668 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-03 14:47:57,624 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-03 14:47:57,631 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-03 14:47:57,633 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-03 14:47:57,634 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,643 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-03 14:47:57,655 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,658 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,658 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,659 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,661 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,661 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,662 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,662 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,663 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,665 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,666 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,667 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,668 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,669 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,670 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 14:47:57,670 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,671 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-03 14:47:57,673 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 14:47:57,674 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-03 14:47:57,676 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-03 14:47:57,677 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-03 14:47:57,679 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-03 14:47:57,679 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-03 14:47:57,687 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-03 14:47:57,690 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-03 14:47:57,691 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-03 14:47:57,692 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-03 14:47:57,692 main DEBUG createAppenders(={Console})
2024-12-03 14:47:57,693 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-03 14:47:57,693 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-03 14:47:57,693 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-03 14:47:57,693 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-03 14:47:57,694 main DEBUG OutputStream closed
2024-12-03 14:47:57,694 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-03 14:47:57,694 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-03 14:47:57,694 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-03 14:47:57,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-03 14:47:57,772 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-03 14:47:57,774 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-03 14:47:57,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-03 14:47:57,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-03 14:47:57,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-03 14:47:57,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-03 14:47:57,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-03 14:47:57,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-03 14:47:57,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-03 14:47:57,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-03 14:47:57,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-03 14:47:57,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-03 14:47:57,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-03 14:47:57,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-03 14:47:57,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-03 14:47:57,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-03 14:47:57,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-03 14:47:57,783 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-03 14:47:57,783 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-03 14:47:57,783 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-03 14:47:57,784 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-03T14:47:58,050 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572
2024-12-03 14:47:58,053 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-03 14:47:58,054 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-03T14:47:58,062 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-03T14:47:58,101 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=416, ProcessCount=11, AvailableMemoryMB=6875
2024-12-03T14:47:58,106 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T14:47:58,120 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf, deleteOnExit=true
2024-12-03T14:47:58,120 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-03T14:47:58,121 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/test.cache.data in system properties and HBase conf
2024-12-03T14:47:58,122 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T14:47:58,122 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir in system properties and HBase conf
2024-12-03T14:47:58,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T14:47:58,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T14:47:58,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-03T14:47:58,224 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-03T14:47:58,345 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T14:47:58,350 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:47:58,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:47:58,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T14:47:58,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:47:58,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T14:47:58,354 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T14:47:58,354 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:47:58,354 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:47:58,355 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T14:47:58,355 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/nfs.dump.dir in system properties and HBase conf
2024-12-03T14:47:58,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/java.io.tmpdir in system properties and HBase conf
2024-12-03T14:47:58,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:47:58,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T14:47:58,357 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T14:47:58,851 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:47:59,183 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-03T14:47:59,259 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2565ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-03T14:47:59,346 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:47:59,414 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:47:59,435 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:47:59,436 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:47:59,437 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:47:59,449 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:47:59,452 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:47:59,453 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:47:59,664 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/java.io.tmpdir/jetty-localhost-40375-hadoop-hdfs-3_4_1-tests_jar-_-any-15097721193499206002/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:47:59,670 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:40375}
2024-12-03T14:47:59,671 INFO  [Time-limited test {}] server.Server(415): Started @2977ms
2024-12-03T14:47:59,704 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:48:00,148 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:48:00,163 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:48:00,165 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:48:00,165 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:48:00,166 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:48:00,169 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:48:00,170 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:48:00,311 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/java.io.tmpdir/jetty-localhost-33191-hadoop-hdfs-3_4_1-tests_jar-_-any-10875330625972345753/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:48:00,312 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:33191}
2024-12-03T14:48:00,313 INFO  [Time-limited test {}] server.Server(415): Started @3619ms
2024-12-03T14:48:00,390 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:48:00,551 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:48:00,563 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:48:00,568 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:48:00,569 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:48:00,569 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:48:00,571 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:48:00,572 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:48:00,712 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/java.io.tmpdir/jetty-localhost-37955-hadoop-hdfs-3_4_1-tests_jar-_-any-41248710769264754/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:48:00,713 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:37955}
2024-12-03T14:48:00,713 INFO  [Time-limited test {}] server.Server(415): Started @4019ms
2024-12-03T14:48:00,715 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:48:00,895 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data2/current/BP-598383475-172.17.0.2-1733237278962/current, will proceed with Du for space computation calculation, 
2024-12-03T14:48:00,895 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data3/current/BP-598383475-172.17.0.2-1733237278962/current, will proceed with Du for space computation calculation, 
2024-12-03T14:48:00,895 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data4/current/BP-598383475-172.17.0.2-1733237278962/current, will proceed with Du for space computation calculation, 
2024-12-03T14:48:00,895 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data1/current/BP-598383475-172.17.0.2-1733237278962/current, will proceed with Du for space computation calculation, 
2024-12-03T14:48:00,979 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:48:00,980 WARN  [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:48:01,043 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec49621be2b54aa6 with lease ID 0x7cb8a08abe8e969b: Processing first storage report for DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de from datanode DatanodeRegistration(127.0.0.1:38699, datanodeUuid=dff57559-aca9-4cb7-8321-df80f685e4cf, infoPort=42795, infoSecurePort=0, ipcPort=39349, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963)
2024-12-03T14:48:01,044 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec49621be2b54aa6 with lease ID 0x7cb8a08abe8e969b: from storage DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de node DatanodeRegistration(127.0.0.1:38699, datanodeUuid=dff57559-aca9-4cb7-8321-df80f685e4cf, infoPort=42795, infoSecurePort=0, ipcPort=39349, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-12-03T14:48:01,045 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafcffbc34db7aeca with lease ID 0x7cb8a08abe8e969c: Processing first storage report for DS-f3c60590-e6e4-4b24-9fb6-7721931df60d from datanode DatanodeRegistration(127.0.0.1:33841, datanodeUuid=a7e01857-2e5f-4b87-a2b4-7fda81e8d6ef, infoPort=37169, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963)
2024-12-03T14:48:01,045 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafcffbc34db7aeca with lease ID 0x7cb8a08abe8e969c: from storage DS-f3c60590-e6e4-4b24-9fb6-7721931df60d node DatanodeRegistration(127.0.0.1:33841, datanodeUuid=a7e01857-2e5f-4b87-a2b4-7fda81e8d6ef, infoPort=37169, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:48:01,045 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec49621be2b54aa6 with lease ID 0x7cb8a08abe8e969b: Processing first storage report for DS-df981a9c-3418-4499-a9f2-1ec3170d9075 from datanode DatanodeRegistration(127.0.0.1:38699, datanodeUuid=dff57559-aca9-4cb7-8321-df80f685e4cf, infoPort=42795, infoSecurePort=0, ipcPort=39349, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963)
2024-12-03T14:48:01,045 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec49621be2b54aa6 with lease ID 0x7cb8a08abe8e969b: from storage DS-df981a9c-3418-4499-a9f2-1ec3170d9075 node DatanodeRegistration(127.0.0.1:38699, datanodeUuid=dff57559-aca9-4cb7-8321-df80f685e4cf, infoPort=42795, infoSecurePort=0, ipcPort=39349, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:48:01,046 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafcffbc34db7aeca with lease ID 0x7cb8a08abe8e969c: Processing first storage report for DS-5cb9ba8d-68b5-4719-a2b2-5d63e8227748 from datanode DatanodeRegistration(127.0.0.1:33841, datanodeUuid=a7e01857-2e5f-4b87-a2b4-7fda81e8d6ef, infoPort=37169, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963)
2024-12-03T14:48:01,046 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafcffbc34db7aeca with lease ID 0x7cb8a08abe8e969c: from storage DS-5cb9ba8d-68b5-4719-a2b2-5d63e8227748 node DatanodeRegistration(127.0.0.1:33841, datanodeUuid=a7e01857-2e5f-4b87-a2b4-7fda81e8d6ef, infoPort=37169, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=397055711;c=1733237278963), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:48:01,100 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572
2024-12-03T14:48:01,172 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/zookeeper_0, clientPort=51937, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T14:48:01,181 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51937
2024-12-03T14:48:01,190 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:01,192 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:01,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:48:01,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:48:01,828 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea with version=8
2024-12-03T14:48:01,828 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase-staging
2024-12-03T14:48:01,912 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-03T14:48:02,139 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:48:02,151 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:48:02,151 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:48:02,155 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:48:02,155 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:48:02,156 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:48:02,309 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-03T14:48:02,371 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-03T14:48:02,382 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-03T14:48:02,387 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:48:02,420 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 90303 (auto-detected)
2024-12-03T14:48:02,421 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-03T14:48:02,444 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34929
2024-12-03T14:48:02,469 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34929 connecting to ZooKeeper ensemble=127.0.0.1:51937
2024-12-03T14:48:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349290x0, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:48:02,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34929-0x100a08d6df10000 connected
2024-12-03T14:48:02,530 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:02,533 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:02,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:48:02,552 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea, hbase.cluster.distributed=false
2024-12-03T14:48:02,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:48:02,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34929
2024-12-03T14:48:02,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34929
2024-12-03T14:48:02,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34929
2024-12-03T14:48:02,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34929
2024-12-03T14:48:02,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34929
2024-12-03T14:48:02,731 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:48:02,734 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:48:02,735 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:48:02,735 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:48:02,735 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:48:02,735 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:48:02,739 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:48:02,741 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:48:02,742 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38513
2024-12-03T14:48:02,745 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38513 connecting to ZooKeeper ensemble=127.0.0.1:51937
2024-12-03T14:48:02,747 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:02,754 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:02,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385130x0, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:48:02,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385130x0, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:48:02,778 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:48:02,782 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38513-0x100a08d6df10001 connected
2024-12-03T14:48:02,787 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:48:02,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T14:48:02,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:48:02,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38513
2024-12-03T14:48:02,801 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38513
2024-12-03T14:48:02,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38513
2024-12-03T14:48:02,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38513
2024-12-03T14:48:02,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38513
2024-12-03T14:48:02,826 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:34929
2024-12-03T14:48:02,827 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:02,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:48:02,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:48:02,834 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:02,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T14:48:02,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:02,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:02,860 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:48:02,861 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,34929,1733237281965 from backup master directory
2024-12-03T14:48:02,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:02,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:48:02,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:48:02,864 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:48:02,864 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:02,866 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-03T14:48:02,868 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-03T14:48:02,925 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase.id] with ID: a7691f04-586b-4ca8-bf08-4db21f43820e
2024-12-03T14:48:02,925 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/.tmp/hbase.id
2024-12-03T14:48:02,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:48:02,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:48:02,941 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/.tmp/hbase.id]:[hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase.id]
2024-12-03T14:48:02,993 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:02,998 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-03T14:48:03,019 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-12-03T14:48:03,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:48:03,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:48:03,059 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:48:03,062 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T14:48:03,069 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:48:03,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:48:03,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:48:03,137 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store
2024-12-03T14:48:03,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:48:03,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:48:03,171 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-03T14:48:03,175 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:48:03,177 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:48:03,177 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:48:03,177 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:48:03,179 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:48:03,179 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:48:03,180 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:48:03,182 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237283177Disabling compacts and flushes for region at 1733237283177Disabling writes for close at 1733237283179 (+2 ms)Writing region close event to WAL at 1733237283180 (+1 ms)Closed at 1733237283180
2024-12-03T14:48:03,185 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/.initializing
2024-12-03T14:48:03,185 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/WALs/a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:03,214 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C34929%2C1733237281965, suffix=, logDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/WALs/a5d22df9eca2,34929,1733237281965, archiveDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/oldWALs, maxLogs=10
2024-12-03T14:48:03,233 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34929%2C1733237281965.1733237283224
2024-12-03T14:48:03,269 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/WALs/a5d22df9eca2,34929,1733237281965/a5d22df9eca2%2C34929%2C1733237281965.1733237283224
2024-12-03T14:48:03,285 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37169:37169),(127.0.0.1/127.0.0.1:42795:42795)]
2024-12-03T14:48:03,292 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:48:03,292 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:48:03,295 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,296 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,339 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:48:03,367 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T14:48:03,370 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,372 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:03,373 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:48:03,375 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T14:48:03,376 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,377 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:48:03,377 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:48:03,380 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T14:48:03,381 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,382 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:48:03,383 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:48:03,387 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T14:48:03,388 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,389 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:48:03,390 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,395 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,396 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,401 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,402 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,406 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T14:48:03,411 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:48:03,422 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:48:03,424 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826146, jitterRate=0.05049969255924225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T14:48:03,431 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733237283311Initializing all the Stores at 1733237283313 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237283314 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237283314Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237283315 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237283315Cleaning up temporary data from old regions at 1733237283402 (+87 ms)Region opened successfully at 1733237283431 (+29 ms)
2024-12-03T14:48:03,433 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T14:48:03,468 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b47bd58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:48:03,497 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-03T14:48:03,508 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T14:48:03,508 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T14:48:03,512 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T14:48:03,514 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec
2024-12-03T14:48:03,519 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-12-03T14:48:03,519 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T14:48:03,544 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T14:48:03,552 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T14:48:03,553 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-03T14:48:03,556 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T14:48:03,559 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T14:48:03,560 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-03T14:48:03,563 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T14:48:03,568 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T14:48:03,569 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-03T14:48:03,571 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T14:48:03,572 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T14:48:03,591 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T14:48:03,592 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T14:48:03,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:48:03,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:48:03,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,599 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,34929,1733237281965, sessionid=0x100a08d6df10000, setting cluster-up flag (Was=false)
2024-12-03T14:48:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,617 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T14:48:03,619 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:03,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:03,637 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T14:48:03,639 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:03,646 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-03T14:48:03,712 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(746): ClusterId : a7691f04-586b-4ca8-bf08-4db21f43820e
2024-12-03T14:48:03,714 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:48:03,720 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:48:03,720 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:48:03,723 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:48:03,724 DEBUG [RS:0;a5d22df9eca2:38513 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f5b4f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:48:03,726 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-03T14:48:03,734 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-03T14:48:03,737 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:38513
2024-12-03T14:48:03,739 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T14:48:03,740 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:48:03,741 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:48:03,741 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:48:03,743 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,34929,1733237281965 with port=38513, startcode=1733237282685
2024-12-03T14:48:03,745 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,34929,1733237281965
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T14:48:03,754 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:48:03,755 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:48:03,755 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:48:03,755 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:48:03,755 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10
2024-12-03T14:48:03,755 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,756 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:48:03,756 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,757 DEBUG [RS:0;a5d22df9eca2:38513 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:48:03,762 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237313762
2024-12-03T14:48:03,763 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T14:48:03,764 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T14:48:03,766 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:48:03,767 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-03T14:48:03,768 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T14:48:03,768 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T14:48:03,769 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T14:48:03,769 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T14:48:03,770 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,773 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,773 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T14:48:03,773 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T14:48:03,774 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T14:48:03,775 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T14:48:03,779 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T14:48:03,779 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T14:48:03,783 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237283781,5,FailOnTimeoutGroup]
2024-12-03T14:48:03,786 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237283783,5,FailOnTimeoutGroup]
2024-12-03T14:48:03,786 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,787 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T14:48:03,788 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,789 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:48:03,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:48:03,802 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-03T14:48:03,803 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea
2024-12-03T14:48:03,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:48:03,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:48:03,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:48:03,839 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:48:03,845 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:48:03,849 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:48:03,849 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,850 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:03,851 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34929 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:03,851 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:48:03,853 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34929 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:03,854 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:48:03,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,855 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:03,856 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:48:03,862 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:48:03,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,864 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:03,864 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:48:03,867 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:48:03,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:03,869 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:03,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:48:03,870 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea
2024-12-03T14:48:03,870 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43675
2024-12-03T14:48:03,871 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:48:03,871 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740
2024-12-03T14:48:03,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740
2024-12-03T14:48:03,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:48:03,877 DEBUG [RS:0;a5d22df9eca2:38513 {}] zookeeper.ZKUtil(111): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:03,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:48:03,877 WARN  [RS:0;a5d22df9eca2:38513 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:48:03,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:48:03,877 INFO  [RS:0;a5d22df9eca2:38513 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:48:03,878 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:03,878 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:48:03,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:48:03,884 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,38513,1733237282685]
2024-12-03T14:48:03,886 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:48:03,887 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=866506, jitterRate=0.10182026028633118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:48:03,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733237283839Initializing all the Stores at 1733237283841 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237283841Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237283845 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237283845Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237283845Cleaning up temporary data from old regions at 1733237283877 (+32 ms)Region opened successfully at 1733237283890 (+13 ms)
2024-12-03T14:48:03,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:48:03,891 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:48:03,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:48:03,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:48:03,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:48:03,894 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:48:03,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237283891Disabling compacts and flushes for region at 1733237283891Disabling writes for close at 1733237283891Writing region close event to WAL at 1733237283893 (+2 ms)Closed at 1733237283894 (+1 ms)
2024-12-03T14:48:03,897 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:48:03,898 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-03T14:48:03,904 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T14:48:03,914 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:48:03,914 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:48:03,917 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T14:48:03,928 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:48:03,933 INFO  [RS:0;a5d22df9eca2:38513 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:48:03,933 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,935 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:48:03,941 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:48:03,943 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,943 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,943 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,944 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,944 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,944 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,944 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:48:03,944 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,945 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,945 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,945 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,945 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,945 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:48:03,946 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:48:03,946 DEBUG [RS:0;a5d22df9eca2:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:48:03,949 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,949 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,949 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,949 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,950 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,950 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,38513,1733237282685-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:48:03,972 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:48:03,974 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,38513,1733237282685-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,975 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,975 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.Replication(171): a5d22df9eca2,38513,1733237282685 started
2024-12-03T14:48:03,995 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:03,996 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,38513,1733237282685, RpcServer on a5d22df9eca2/172.17.0.2:38513, sessionid=0x100a08d6df10001
2024-12-03T14:48:03,997 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:48:03,997 DEBUG [RS:0;a5d22df9eca2:38513 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:03,997 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,38513,1733237282685'
2024-12-03T14:48:03,997 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:48:03,999 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:48:03,999 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:48:04,000 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:48:04,000 DEBUG [RS:0;a5d22df9eca2:38513 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:04,000 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,38513,1733237282685'
2024-12-03T14:48:04,000 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:48:04,002 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:48:04,003 DEBUG [RS:0;a5d22df9eca2:38513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:48:04,003 INFO  [RS:0;a5d22df9eca2:38513 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:48:04,003 INFO  [RS:0;a5d22df9eca2:38513 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:48:04,068 WARN  [a5d22df9eca2:34929 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-03T14:48:04,111 INFO  [RS:0;a5d22df9eca2:38513 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C38513%2C1733237282685, suffix=, logDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685, archiveDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs, maxLogs=32
2024-12-03T14:48:04,114 INFO  [RS:0;a5d22df9eca2:38513 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237284113
2024-12-03T14:48:04,130 INFO  [RS:0;a5d22df9eca2:38513 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237284113
2024-12-03T14:48:04,132 DEBUG [RS:0;a5d22df9eca2:38513 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42795:42795),(127.0.0.1/127.0.0.1:37169:37169)]
2024-12-03T14:48:04,321 DEBUG [a5d22df9eca2:34929 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T14:48:04,336 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:04,343 INFO  [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,38513,1733237282685, state=OPENING
2024-12-03T14:48:04,348 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T14:48:04,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:04,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:48:04,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:48:04,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:48:04,353 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:48:04,355 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,38513,1733237282685}]
2024-12-03T14:48:04,536 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T14:48:04,539 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52435, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T14:48:04,555 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-03T14:48:04,556 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:48:04,560 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C38513%2C1733237282685.meta, suffix=.meta, logDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685, archiveDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs, maxLogs=32
2024-12-03T14:48:04,562 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.meta.1733237284562.meta
2024-12-03T14:48:04,583 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.meta.1733237284562.meta
2024-12-03T14:48:04,584 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42795:42795),(127.0.0.1/127.0.0.1:37169:37169)]
2024-12-03T14:48:04,586 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:48:04,588 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T14:48:04,592 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T14:48:04,597 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T14:48:04,601 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T14:48:04,601 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:48:04,602 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-03T14:48:04,602 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-03T14:48:04,605 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:48:04,606 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:48:04,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:04,608 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:04,608 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:48:04,610 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:48:04,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:04,612 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:04,612 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:48:04,615 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:48:04,615 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:04,616 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:04,616 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:48:04,618 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:48:04,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:04,620 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:48:04,621 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:48:04,623 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740
2024-12-03T14:48:04,628 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740
2024-12-03T14:48:04,631 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:48:04,631 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:48:04,632 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:48:04,635 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:48:04,637 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=709869, jitterRate=-0.09735572338104248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:48:04,637 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-03T14:48:04,639 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733237284602Writing region info on filesystem at 1733237284603 (+1 ms)Initializing all the Stores at 1733237284604 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237284604Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237284605 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237284605Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237284605Cleaning up temporary data from old regions at 1733237284631 (+26 ms)Running coprocessor post-open hooks at 1733237284637 (+6 ms)Region opened successfully at 1733237284639 (+2 ms)
2024-12-03T14:48:04,653 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237284525
2024-12-03T14:48:04,665 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T14:48:04,666 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-03T14:48:04,667 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:04,671 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,38513,1733237282685, state=OPEN
2024-12-03T14:48:04,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:48:04,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:48:04,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:48:04,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:48:04,675 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:04,682 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T14:48:04,682 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,38513,1733237282685 in 321 msec
2024-12-03T14:48:04,692 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T14:48:04,692 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 781 msec
2024-12-03T14:48:04,694 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:48:04,694 INFO  [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-03T14:48:04,716 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:48:04,718 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,38513,1733237282685, seqNum=-1]
2024-12-03T14:48:04,739 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:48:04,741 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:48:04,769 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0900 sec
2024-12-03T14:48:04,773 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237284773, completionTime=-1
2024-12-03T14:48:04,776 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T14:48:04,777 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-03T14:48:04,813 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-03T14:48:04,813 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237344813
2024-12-03T14:48:04,813 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237404813
2024-12-03T14:48:04,813 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec
2024-12-03T14:48:04,816 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34929,1733237281965-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:04,817 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34929,1733237281965-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:04,817 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34929,1733237281965-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:04,818 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:34929, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:04,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:04,820 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:04,827 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-03T14:48:04,853 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.987sec
2024-12-03T14:48:04,854 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T14:48:04,856 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T14:48:04,858 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T14:48:04,859 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T14:48:04,859 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T14:48:04,860 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34929,1733237281965-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:48:04,861 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34929,1733237281965-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T14:48:04,869 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-03T14:48:04,870 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T14:48:04,871 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34929,1733237281965-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:48:04,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b598c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:48:04,932 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-03T14:48:04,932 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-03T14:48:04,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,34929,-1 for getting cluster id
2024-12-03T14:48:04,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-03T14:48:04,951 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7691f04-586b-4ca8-bf08-4db21f43820e'
2024-12-03T14:48:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-03T14:48:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7691f04-586b-4ca8-bf08-4db21f43820e"

2024-12-03T14:48:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29842d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:48:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,34929,-1]
2024-12-03T14:48:04,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-03T14:48:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:48:04,964 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-03T14:48:04,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e15a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:48:04,968 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:48:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,38513,1733237282685, seqNum=-1]
2024-12-03T14:48:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:48:04,980 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:48:05,004 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:05,005 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:48:05,043 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-03T14:48:05,048 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-03T14:48:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a5d22df9eca2,34929,1733237281965
2024-12-03T14:48:05,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c3c746e
2024-12-03T14:48:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T14:48:05,072 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T14:48:05,075 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T14:48:05,075 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T14:48:05,082 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:48:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-03T14:48:05,093 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T14:48:05,095 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-03T14:48:05,095 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:05,098 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T14:48:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:48:05,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741835_1011 (size=389)
2024-12-03T14:48:05,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741835_1011 (size=389)
2024-12-03T14:48:05,144 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 92b79b04aac2d4d4a1f6bada64339fdb, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea
2024-12-03T14:48:05,154 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741836_1012 (size=72)
2024-12-03T14:48:05,154 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741836_1012 (size=72)
2024-12-03T14:48:05,155 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:48:05,155 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 92b79b04aac2d4d4a1f6bada64339fdb, disabling compactions & flushes
2024-12-03T14:48:05,155 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:05,155 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:05,155 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb. after waiting 0 ms
2024-12-03T14:48:05,155 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:05,155 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:05,156 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 92b79b04aac2d4d4a1f6bada64339fdb:
Waiting for close lock at 1733237285155Disabling compacts and flushes for region at 1733237285155Disabling writes for close at 1733237285155Writing region close event to WAL at 1733237285155Closed at 1733237285155
2024-12-03T14:48:05,158 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T14:48:05,163 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733237285158"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237285158"}]},"ts":"1733237285158"}
2024-12-03T14:48:05,170 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-03T14:48:05,172 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T14:48:05,176 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237285173"}]},"ts":"1733237285173"}
2024-12-03T14:48:05,181 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-03T14:48:05,183 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92b79b04aac2d4d4a1f6bada64339fdb, ASSIGN}]
2024-12-03T14:48:05,186 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92b79b04aac2d4d4a1f6bada64339fdb, ASSIGN
2024-12-03T14:48:05,188 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92b79b04aac2d4d4a1f6bada64339fdb, ASSIGN; state=OFFLINE, location=a5d22df9eca2,38513,1733237282685; forceNewPlan=false, retain=false
2024-12-03T14:48:05,339 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=92b79b04aac2d4d4a1f6bada64339fdb, regionState=OPENING, regionLocation=a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:05,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92b79b04aac2d4d4a1f6bada64339fdb, ASSIGN because future has completed
2024-12-03T14:48:05,346 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92b79b04aac2d4d4a1f6bada64339fdb, server=a5d22df9eca2,38513,1733237282685}]
2024-12-03T14:48:05,508 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:05,508 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 92b79b04aac2d4d4a1f6bada64339fdb, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:48:05,508 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,509 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:48:05,509 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,509 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,513 INFO  [StoreOpener-92b79b04aac2d4d4a1f6bada64339fdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 92b79b04aac2d4d4a1f6bada64339fdb 
2024-12-03T14:48:05,520 INFO  [StoreOpener-92b79b04aac2d4d4a1f6bada64339fdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92b79b04aac2d4d4a1f6bada64339fdb columnFamilyName info
2024-12-03T14:48:05,520 DEBUG [StoreOpener-92b79b04aac2d4d4a1f6bada64339fdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:48:05,521 INFO  [StoreOpener-92b79b04aac2d4d4a1f6bada64339fdb-1 {}] regionserver.HStore(327): Store=92b79b04aac2d4d4a1f6bada64339fdb/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:48:05,521 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,523 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,523 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,525 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,525 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,528 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,532 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:48:05,533 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 92b79b04aac2d4d4a1f6bada64339fdb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=730813, jitterRate=-0.07072383165359497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T14:48:05,533 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:05,534 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 92b79b04aac2d4d4a1f6bada64339fdb:
Running coprocessor pre-open hook at 1733237285509Writing region info on filesystem at 1733237285509Initializing all the Stores at 1733237285511 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237285511Cleaning up temporary data from old regions at 1733237285525 (+14 ms)Running coprocessor post-open hooks at 1733237285533 (+8 ms)Region opened successfully at 1733237285534 (+1 ms)
2024-12-03T14:48:05,537 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb., pid=6, masterSystemTime=1733237285500
2024-12-03T14:48:05,541 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:05,541 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:05,543 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=92b79b04aac2d4d4a1f6bada64339fdb, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,38513,1733237282685
2024-12-03T14:48:05,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92b79b04aac2d4d4a1f6bada64339fdb, server=a5d22df9eca2,38513,1733237282685 because future has completed
2024-12-03T14:48:05,554 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T14:48:05,555 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 92b79b04aac2d4d4a1f6bada64339fdb, server=a5d22df9eca2,38513,1733237282685 in 205 msec
2024-12-03T14:48:05,559 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T14:48:05,559 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92b79b04aac2d4d4a1f6bada64339fdb, ASSIGN in 372 msec
2024-12-03T14:48:05,560 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T14:48:05,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237285560"}]},"ts":"1733237285560"}
2024-12-03T14:48:05,564 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-03T14:48:05,567 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T14:48:05,571 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 481 msec
2024-12-03T14:48:10,105 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-03T14:48:10,157 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T14:48:10,160 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-03T14:48:12,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-03T14:48:12,367 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-03T14:48:12,369 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-03T14:48:12,369 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-03T14:48:12,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:48:12,370 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-03T14:48:12,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-03T14:48:12,370 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-03T14:48:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:48:15,134 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-03T14:48:15,136 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-03T14:48:15,144 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-03T14:48:15,145 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:48:15,146 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237295145
2024-12-03T14:48:15,160 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:15,160 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:15,160 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:15,161 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:15,161 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:15,161 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237284113 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237295145
2024-12-03T14:48:15,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741833_1009 (size=451)
2024-12-03T14:48:15,167 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741833_1009 (size=451)
2024-12-03T14:48:15,173 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237284113 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs/a5d22df9eca2%2C38513%2C1733237282685.1733237284113
2024-12-03T14:48:15,173 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42795:42795),(127.0.0.1/127.0.0.1:37169:37169)]
2024-12-03T14:48:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb., hostname=a5d22df9eca2,38513,1733237282685, seqNum=2]
2024-12-03T14:48:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:27,232 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92b79b04aac2d4d4a1f6bada64339fdb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:48:27,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/24ca2e48d14b4bd797f4d99857f5af08 is 1080, key is row0001/info:/1733237295186/Put/seqid=0
2024-12-03T14:48:27,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741838_1014 (size=12509)
2024-12-03T14:48:27,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741838_1014 (size=12509)
2024-12-03T14:48:27,329 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/24ca2e48d14b4bd797f4d99857f5af08
2024-12-03T14:48:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/24ca2e48d14b4bd797f4d99857f5af08 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08
2024-12-03T14:48:27,397 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08, entries=7, sequenceid=11, filesize=12.2 K
2024-12-03T14:48:27,409 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92b79b04aac2d4d4a1f6bada64339fdb in 175ms, sequenceid=11, compaction requested=false
2024-12-03T14:48:27,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92b79b04aac2d4d4a1f6bada64339fdb:

2024-12-03T14:48:31,098 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T14:48:35,244 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237315244
2024-12-03T14:48:35,453 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:35,454 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:35,454 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:35,454 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:35,454 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:35,454 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:35,455 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237295145 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237315244
2024-12-03T14:48:35,455 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42795:42795),(127.0.0.1/127.0.0.1:37169:37169)]
2024-12-03T14:48:35,456 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237295145 is not closed yet, will try archiving it next time
2024-12-03T14:48:35,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741837_1013 (size=12399)
2024-12-03T14:48:35,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741837_1013 (size=12399)
2024-12-03T14:48:35,659 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:37,863 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:40,067 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:42,271 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:48:42,272 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92b79b04aac2d4d4a1f6bada64339fdb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:48:42,475 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:42,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/818eb754827e4909957885f9dd347c27 is 1080, key is row0008/info:/1733237309230/Put/seqid=0
2024-12-03T14:48:42,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741840_1016 (size=12509)
2024-12-03T14:48:42,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741840_1016 (size=12509)
2024-12-03T14:48:42,495 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/818eb754827e4909957885f9dd347c27
2024-12-03T14:48:42,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/818eb754827e4909957885f9dd347c27 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/818eb754827e4909957885f9dd347c27
2024-12-03T14:48:42,514 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/818eb754827e4909957885f9dd347c27, entries=7, sequenceid=21, filesize=12.2 K
2024-12-03T14:48:42,716 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:42,716 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92b79b04aac2d4d4a1f6bada64339fdb in 445ms, sequenceid=21, compaction requested=false
2024-12-03T14:48:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92b79b04aac2d4d4a1f6bada64339fdb:

2024-12-03T14:48:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-03T14:48:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:48:42,717 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08 because midkey is the same as first or last row
2024-12-03T14:48:44,478 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:44,874 INFO  [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T14:48:44,874 INFO  [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T14:48:46,685 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:46,688 WARN  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:46,690 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C38513%2C1733237282685:(num 1733237315244) roll requested
2024-12-03T14:48:46,690 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237326690
2024-12-03T14:48:46,903 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 210 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:48:46,904 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:46,904 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:46,904 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:46,904 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:46,905 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:48:46,905 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237315244 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237326690
2024-12-03T14:48:46,906 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37169:37169),(127.0.0.1/127.0.0.1:42795:42795)]
2024-12-03T14:48:46,906 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237315244 is not closed yet, will try archiving it next time
2024-12-03T14:48:46,907 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237295145 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs/a5d22df9eca2%2C38513%2C1733237282685.1733237295145
2024-12-03T14:48:46,908 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741839_1015 (size=7739)
2024-12-03T14:48:46,908 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741839_1015 (size=7739)
2024-12-03T14:48:48,892 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK], DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK]]
2024-12-03T14:48:50,509 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 92b79b04aac2d4d4a1f6bada64339fdb, had cached 0 bytes from a total of 25018
2024-12-03T14:48:51,099 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK], DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK]]
2024-12-03T14:48:53,304 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK], DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK]]
2024-12-03T14:48:55,513 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK], DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK]]
2024-12-03T14:48:57,516 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-03T14:48:57,516 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237337516
2024-12-03T14:49:01,098 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T14:49:02,531 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5011 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK], DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK]]
2024-12-03T14:49:02,534 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5011 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK], DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK]]
2024-12-03T14:49:02,534 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C38513%2C1733237282685:(num 1733237337516) roll requested
2024-12-03T14:49:02,534 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:02,535 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:02,535 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:02,535 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:02,535 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:02,536 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237326690 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237337516
2024-12-03T14:49:02,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42795:42795),(127.0.0.1/127.0.0.1:37169:37169)]
2024-12-03T14:49:02,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237326690 is not closed yet, will try archiving it next time
2024-12-03T14:49:02,539 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237342538
2024-12-03T14:49:02,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741841_1017 (size=4753)
2024-12-03T14:49:02,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741841_1017 (size=4753)
2024-12-03T14:49:07,543 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:07,543 WARN  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:49:07,544 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92b79b04aac2d4d4a1f6bada64339fdb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:49:07,551 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:07,552 WARN  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:09,544 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-03T14:49:12,549 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:12,549 WARN  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:12,550 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:12,550 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:12,551 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:12,551 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:12,552 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:12,552 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237337516 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237342538
2024-12-03T14:49:12,555 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42795:42795),(127.0.0.1/127.0.0.1:37169:37169)]
2024-12-03T14:49:12,555 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237337516 is not closed yet, will try archiving it next time
2024-12-03T14:49:12,555 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C38513%2C1733237282685:(num 1733237342538) roll requested
2024-12-03T14:49:12,556 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237352555
2024-12-03T14:49:12,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741842_1018 (size=1569)
2024-12-03T14:49:12,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741842_1018 (size=1569)
2024-12-03T14:49:12,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/d2b30a852a9047d5af5e26e3d0efbcd1 is 1080, key is row0015/info:/1733237324275/Put/seqid=0
2024-12-03T14:49:12,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741844_1020 (size=12509)
2024-12-03T14:49:12,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741844_1020 (size=12509)
2024-12-03T14:49:12,566 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/d2b30a852a9047d5af5e26e3d0efbcd1
2024-12-03T14:49:12,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/d2b30a852a9047d5af5e26e3d0efbcd1 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/d2b30a852a9047d5af5e26e3d0efbcd1
2024-12-03T14:49:12,585 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/d2b30a852a9047d5af5e26e3d0efbcd1, entries=7, sequenceid=31, filesize=12.2 K
2024-12-03T14:49:17,568 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:17,568 WARN  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:17,587 INFO  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:17,587 WARN  [FSHLog-0-hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea-prefix:a5d22df9eca2,38513,1733237282685 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38699,DS-6f0e4b80-8423-47a8-bd1f-57bc7f6928de,DISK], DatanodeInfoWithStorage[127.0.0.1:33841,DS-f3c60590-e6e4-4b24-9fb6-7721931df60d,DISK]]
2024-12-03T14:49:17,587 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92b79b04aac2d4d4a1f6bada64339fdb in 10044ms, sequenceid=31, compaction requested=true
2024-12-03T14:49:17,587 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92b79b04aac2d4d4a1f6bada64339fdb:

2024-12-03T14:49:17,587 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,588 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,588 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-03T14:49:17,588 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:49:17,588 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,588 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08 because midkey is the same as first or last row
2024-12-03T14:49:17,588 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,588 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237342538 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237352555
2024-12-03T14:49:17,590 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37169:37169),(127.0.0.1/127.0.0.1:42795:42795)]
2024-12-03T14:49:17,590 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237342538 is not closed yet, will try archiving it next time
2024-12-03T14:49:17,590 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237315244 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs/a5d22df9eca2%2C38513%2C1733237282685.1733237315244
2024-12-03T14:49:17,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92b79b04aac2d4d4a1f6bada64339fdb:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:49:17,591 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C38513%2C1733237282685:(num 1733237352555) roll requested
2024-12-03T14:49:17,591 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237357591
2024-12-03T14:49:17,592 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741843_1019 (size=438)
2024-12-03T14:49:17,593 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237326690 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs/a5d22df9eca2%2C38513%2C1733237282685.1733237326690
2024-12-03T14:49:17,593 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741843_1019 (size=438)
2024-12-03T14:49:17,595 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:49:17,595 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237337516 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs/a5d22df9eca2%2C38513%2C1733237282685.1733237337516
2024-12-03T14:49:17,597 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237342538 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs/a5d22df9eca2%2C38513%2C1733237282685.1733237342538
2024-12-03T14:49:17,598 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:49:17,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:49:17,599 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.HStore(1541): 92b79b04aac2d4d4a1f6bada64339fdb/info is initiating minor compaction (all files)
2024-12-03T14:49:17,600 INFO  [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92b79b04aac2d4d4a1f6bada64339fdb/info in TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:49:17,600 INFO  [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08, hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/818eb754827e4909957885f9dd347c27, hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/d2b30a852a9047d5af5e26e3d0efbcd1] into tmpdir=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp, totalSize=36.6 K
2024-12-03T14:49:17,602 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24ca2e48d14b4bd797f4d99857f5af08, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733237295186
2024-12-03T14:49:17,602 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 818eb754827e4909957885f9dd347c27, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733237309230
2024-12-03T14:49:17,603 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,603 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,603 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,603 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,603 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,603 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b30a852a9047d5af5e26e3d0efbcd1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733237324275
2024-12-03T14:49:17,603 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237352555 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237357591
2024-12-03T14:49:17,605 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37169:37169),(127.0.0.1/127.0.0.1:42795:42795)]
2024-12-03T14:49:17,605 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237352555 is not closed yet, will try archiving it next time
2024-12-03T14:49:17,605 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38513%2C1733237282685.1733237357605
2024-12-03T14:49:17,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741845_1021 (size=93)
2024-12-03T14:49:17,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741845_1021 (size=93)
2024-12-03T14:49:17,607 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237352555 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs/a5d22df9eca2%2C38513%2C1733237282685.1733237352555
2024-12-03T14:49:17,612 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,612 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,612 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,612 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,612 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:17,613 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237357591 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237357605
2024-12-03T14:49:17,614 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42795:42795),(127.0.0.1/127.0.0.1:37169:37169)]
2024-12-03T14:49:17,614 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/WALs/a5d22df9eca2,38513,1733237282685/a5d22df9eca2%2C38513%2C1733237282685.1733237357591 is not closed yet, will try archiving it next time
2024-12-03T14:49:17,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741846_1022 (size=1258)
2024-12-03T14:49:17,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741846_1022 (size=1258)
2024-12-03T14:49:17,633 INFO  [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92b79b04aac2d4d4a1f6bada64339fdb#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:49:17,634 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/ab8c77c95c0a4863b4879adc33d3f094 is 1080, key is row0001/info:/1733237295186/Put/seqid=0
2024-12-03T14:49:17,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741848_1024 (size=27710)
2024-12-03T14:49:17,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741848_1024 (size=27710)
2024-12-03T14:49:17,653 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/ab8c77c95c0a4863b4879adc33d3f094 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/ab8c77c95c0a4863b4879adc33d3f094
2024-12-03T14:49:17,669 INFO  [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92b79b04aac2d4d4a1f6bada64339fdb/info of 92b79b04aac2d4d4a1f6bada64339fdb into ab8c77c95c0a4863b4879adc33d3f094(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:49:17,670 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92b79b04aac2d4d4a1f6bada64339fdb:

2024-12-03T14:49:17,673 INFO  [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb., storeName=92b79b04aac2d4d4a1f6bada64339fdb/info, priority=13, startTime=1733237357590; duration=0sec
2024-12-03T14:49:17,673 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-03T14:49:17,673 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:49:17,673 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/ab8c77c95c0a4863b4879adc33d3f094 because midkey is the same as first or last row
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/ab8c77c95c0a4863b4879adc33d3f094 because midkey is the same as first or last row
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/ab8c77c95c0a4863b4879adc33d3f094 because midkey is the same as first or last row
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:49:17,674 DEBUG [RS:0;a5d22df9eca2:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92b79b04aac2d4d4a1f6bada64339fdb:info
2024-12-03T14:49:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:49:29,645 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92b79b04aac2d4d4a1f6bada64339fdb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:49:29,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/ab202164d6554ba59ad9d6e7a9f1483d is 1080, key is row0022/info:/1733237357607/Put/seqid=0
2024-12-03T14:49:29,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741849_1025 (size=12509)
2024-12-03T14:49:29,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741849_1025 (size=12509)
2024-12-03T14:49:29,663 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/ab202164d6554ba59ad9d6e7a9f1483d
2024-12-03T14:49:29,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/ab202164d6554ba59ad9d6e7a9f1483d as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/ab202164d6554ba59ad9d6e7a9f1483d
2024-12-03T14:49:29,683 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/ab202164d6554ba59ad9d6e7a9f1483d, entries=7, sequenceid=42, filesize=12.2 K
2024-12-03T14:49:29,684 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92b79b04aac2d4d4a1f6bada64339fdb in 39ms, sequenceid=42, compaction requested=false
2024-12-03T14:49:29,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92b79b04aac2d4d4a1f6bada64339fdb:

2024-12-03T14:49:29,685 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-03T14:49:29,685 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:49:29,685 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/ab8c77c95c0a4863b4879adc33d3f094 because midkey is the same as first or last row
2024-12-03T14:49:31,099 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T14:49:35,509 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 92b79b04aac2d4d4a1f6bada64339fdb, had cached 0 bytes from a total of 40219
2024-12-03T14:49:37,669 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-03T14:49:37,671 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:49:37,671 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:49:37,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:37,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:37,680 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-03T14:49:37,680 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T14:49:37,680 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=63992445, stopped=false
2024-12-03T14:49:37,680 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,34929,1733237281965
2024-12-03T14:49:37,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:37,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:37,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:37,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:37,682 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:49:37,682 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:49:37,682 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:49:37,682 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:37,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:37,682 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:37,682 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,38513,1733237282685' *****
2024-12-03T14:49:37,682 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:49:37,683 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:49:37,683 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:49:37,683 INFO  [RS:0;a5d22df9eca2:38513 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:49:37,683 INFO  [RS:0;a5d22df9eca2:38513 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:49:37,683 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(3091): Received CLOSE for 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:49:37,684 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,38513,1733237282685
2024-12-03T14:49:37,684 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:49:37,684 INFO  [RS:0;a5d22df9eca2:38513 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:38513.
2024-12-03T14:49:37,684 DEBUG [RS:0;a5d22df9eca2:38513 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:49:37,684 DEBUG [RS:0;a5d22df9eca2:38513 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:37,684 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 92b79b04aac2d4d4a1f6bada64339fdb, disabling compactions & flushes
2024-12-03T14:49:37,684 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:49:37,684 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:49:37,684 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:49:37,684 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:49:37,684 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:49:37,685 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb. after waiting 0 ms
2024-12-03T14:49:37,685 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-03T14:49:37,685 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:49:37,685 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 92b79b04aac2d4d4a1f6bada64339fdb 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-03T14:49:37,685 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-03T14:49:37,685 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 92b79b04aac2d4d4a1f6bada64339fdb=TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.}
2024-12-03T14:49:37,685 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:49:37,685 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:49:37,685 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:49:37,685 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:49:37,685 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:49:37,685 DEBUG [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 92b79b04aac2d4d4a1f6bada64339fdb
2024-12-03T14:49:37,685 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-03T14:49:37,691 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/88f037455f234fbebcaa0bf431e69b4e is 1080, key is row0029/info:/1733237371650/Put/seqid=0
2024-12-03T14:49:37,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741850_1026 (size=8193)
2024-12-03T14:49:37,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741850_1026 (size=8193)
2024-12-03T14:49:37,698 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/88f037455f234fbebcaa0bf431e69b4e
2024-12-03T14:49:37,707 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/.tmp/info/88f037455f234fbebcaa0bf431e69b4e as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/88f037455f234fbebcaa0bf431e69b4e
2024-12-03T14:49:37,707 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/info/475a298771aa4bc8be043a2c73b75a34 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb./info:regioninfo/1733237285542/Put/seqid=0
2024-12-03T14:49:37,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741851_1027 (size=7016)
2024-12-03T14:49:37,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741851_1027 (size=7016)
2024-12-03T14:49:37,715 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/info/475a298771aa4bc8be043a2c73b75a34
2024-12-03T14:49:37,716 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/88f037455f234fbebcaa0bf431e69b4e, entries=3, sequenceid=48, filesize=8.0 K
2024-12-03T14:49:37,718 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 92b79b04aac2d4d4a1f6bada64339fdb in 33ms, sequenceid=48, compaction requested=true
2024-12-03T14:49:37,718 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08, hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/818eb754827e4909957885f9dd347c27, hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/d2b30a852a9047d5af5e26e3d0efbcd1] to archive
2024-12-03T14:49:37,721 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T14:49:37,725 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/archive/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/24ca2e48d14b4bd797f4d99857f5af08
2024-12-03T14:49:37,727 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/818eb754827e4909957885f9dd347c27 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/archive/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/818eb754827e4909957885f9dd347c27
2024-12-03T14:49:37,729 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/d2b30a852a9047d5af5e26e3d0efbcd1 to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/archive/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/info/d2b30a852a9047d5af5e26e3d0efbcd1
2024-12-03T14:49:37,745 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/ns/7df6ad1bdf2947459938219a239216e2 is 43, key is default/ns:d/1733237284746/Put/seqid=0
2024-12-03T14:49:37,742 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a5d22df9eca2:34929 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-03T14:49:37,746 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [24ca2e48d14b4bd797f4d99857f5af08=12509, 818eb754827e4909957885f9dd347c27=12509, d2b30a852a9047d5af5e26e3d0efbcd1=12509]
2024-12-03T14:49:37,752 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/default/TestLogRolling-testSlowSyncLogRolling/92b79b04aac2d4d4a1f6bada64339fdb/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-03T14:49:37,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741852_1028 (size=5153)
2024-12-03T14:49:37,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741852_1028 (size=5153)
2024-12-03T14:49:37,755 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/ns/7df6ad1bdf2947459938219a239216e2
2024-12-03T14:49:37,755 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:49:37,755 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 92b79b04aac2d4d4a1f6bada64339fdb:
Waiting for close lock at 1733237377684Running coprocessor pre-close hooks at 1733237377684Disabling compacts and flushes for region at 1733237377684Disabling writes for close at 1733237377685 (+1 ms)Obtaining lock to block concurrent updates at 1733237377685Preparing flush snapshotting stores in 92b79b04aac2d4d4a1f6bada64339fdb at 1733237377685Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733237377685Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb. at 1733237377686 (+1 ms)Flushing 92b79b04aac2d4d4a1f6bada64339fdb/info: creating writer at 1733237377686Flushing 92b79b04aac2d4d4a1f6bada64339fdb/info: appending metadata at 1733237377690 (+4 ms)Flushing 92b79b04aac2d4d4a1f6bada64339fdb/info: closing flushed file at 1733237377690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65cfd5bb: reopening flushed file at 1733237377705 (+15 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 92b79b04aac2d4d4a1f6bada64339fdb in 33ms, sequenceid=48, compaction requested=true at 1733237377718 (+13 ms)Writing region close event to WAL at 1733237377747 (+29 ms)Running coprocessor post-close hooks at 1733237377753 (+6 ms)Closed at 1733237377755 (+2 ms)
2024-12-03T14:49:37,756 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733237285075.92b79b04aac2d4d4a1f6bada64339fdb.
2024-12-03T14:49:37,778 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/table/aa4ec1037ac14d2b9dbcc043631241ca is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733237285560/Put/seqid=0
2024-12-03T14:49:37,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741853_1029 (size=5396)
2024-12-03T14:49:37,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741853_1029 (size=5396)
2024-12-03T14:49:37,783 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/table/aa4ec1037ac14d2b9dbcc043631241ca
2024-12-03T14:49:37,791 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/info/475a298771aa4bc8be043a2c73b75a34 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/info/475a298771aa4bc8be043a2c73b75a34
2024-12-03T14:49:37,799 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/info/475a298771aa4bc8be043a2c73b75a34, entries=10, sequenceid=11, filesize=6.9 K
2024-12-03T14:49:37,801 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/ns/7df6ad1bdf2947459938219a239216e2 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/ns/7df6ad1bdf2947459938219a239216e2
2024-12-03T14:49:37,808 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/ns/7df6ad1bdf2947459938219a239216e2, entries=2, sequenceid=11, filesize=5.0 K
2024-12-03T14:49:37,809 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/.tmp/table/aa4ec1037ac14d2b9dbcc043631241ca as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/table/aa4ec1037ac14d2b9dbcc043631241ca
2024-12-03T14:49:37,816 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/table/aa4ec1037ac14d2b9dbcc043631241ca, entries=2, sequenceid=11, filesize=5.3 K
2024-12-03T14:49:37,817 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 132ms, sequenceid=11, compaction requested=false
2024-12-03T14:49:37,823 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-03T14:49:37,824 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:49:37,824 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:49:37,824 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237377685Running coprocessor pre-close hooks at 1733237377685Disabling compacts and flushes for region at 1733237377685Disabling writes for close at 1733237377685Obtaining lock to block concurrent updates at 1733237377685Preparing flush snapshotting stores in 1588230740 at 1733237377685Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733237377686 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733237377686Flushing 1588230740/info: creating writer at 1733237377687 (+1 ms)Flushing 1588230740/info: appending metadata at 1733237377707 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733237377707Flushing 1588230740/ns: creating writer at 1733237377725 (+18 ms)Flushing 1588230740/ns: appending metadata at 1733237377744 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733237377745 (+1 ms)Flushing 1588230740/table: creating writer at 1733237377763 (+18 ms)Flushing 1588230740/table: appending metadata at 1733237377777 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733237377777Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69b48848: reopening flushed file at 1733237377790 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bb98c7b: reopening flushed file at 1733237377800 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@232aaddb: reopening flushed file at 1733237377808 (+8 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 132ms, sequenceid=11, compaction requested=false at 1733237377817 (+9 ms)Writing region close event to WAL at 1733237377819 (+2 ms)Running coprocessor post-close hooks at 1733237377824 (+5 ms)Closed at 1733237377824
2024-12-03T14:49:37,825 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T14:49:37,886 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,38513,1733237282685; all regions closed.
2024-12-03T14:49:37,888 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,889 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,889 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,889 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,890 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,895 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741834_1010 (size=3066)
2024-12-03T14:49:37,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741834_1010 (size=3066)
2024-12-03T14:49:37,900 DEBUG [RS:0;a5d22df9eca2:38513 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs
2024-12-03T14:49:37,900 INFO  [RS:0;a5d22df9eca2:38513 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C38513%2C1733237282685.meta:.meta(num 1733237284562)
2024-12-03T14:49:37,901 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,901 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,901 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,901 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,901 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:37,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741847_1023 (size=12695)
2024-12-03T14:49:37,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741847_1023 (size=12695)
2024-12-03T14:49:37,908 DEBUG [RS:0;a5d22df9eca2:38513 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/oldWALs
2024-12-03T14:49:37,908 INFO  [RS:0;a5d22df9eca2:38513 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C38513%2C1733237282685:(num 1733237357605)
2024-12-03T14:49:37,908 DEBUG [RS:0;a5d22df9eca2:38513 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:37,909 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:49:37,909 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:49:37,909 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-03T14:49:37,909 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:49:37,909 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:49:37,910 INFO  [RS:0;a5d22df9eca2:38513 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38513
2024-12-03T14:49:37,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,38513,1733237282685
2024-12-03T14:49:37,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:49:37,913 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:49:37,914 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,38513,1733237282685]
2024-12-03T14:49:37,915 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,38513,1733237282685 already deleted, retry=false
2024-12-03T14:49:37,916 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,38513,1733237282685 expired; onlineServers=0
2024-12-03T14:49:37,916 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,34929,1733237281965' *****
2024-12-03T14:49:37,916 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T14:49:37,916 INFO  [M:0;a5d22df9eca2:34929 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:49:37,916 INFO  [M:0;a5d22df9eca2:34929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:49:37,916 DEBUG [M:0;a5d22df9eca2:34929 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T14:49:37,916 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T14:49:37,916 DEBUG [M:0;a5d22df9eca2:34929 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T14:49:37,916 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237283783 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237283783,5,FailOnTimeoutGroup]
2024-12-03T14:49:37,916 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237283781 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237283781,5,FailOnTimeoutGroup]
2024-12-03T14:49:37,916 INFO  [M:0;a5d22df9eca2:34929 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-03T14:49:37,916 INFO  [M:0;a5d22df9eca2:34929 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:49:37,916 DEBUG [M:0;a5d22df9eca2:34929 {}] master.HMaster(1795): Stopping service threads
2024-12-03T14:49:37,917 INFO  [M:0;a5d22df9eca2:34929 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T14:49:37,917 INFO  [M:0;a5d22df9eca2:34929 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:49:37,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T14:49:37,917 INFO  [M:0;a5d22df9eca2:34929 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T14:49:37,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:37,917 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T14:49:37,918 DEBUG [M:0;a5d22df9eca2:34929 {}] zookeeper.ZKUtil(347): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T14:49:37,918 WARN  [M:0;a5d22df9eca2:34929 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T14:49:37,919 INFO  [M:0;a5d22df9eca2:34929 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/.lastflushedseqids
2024-12-03T14:49:37,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741854_1030 (size=130)
2024-12-03T14:49:37,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741854_1030 (size=130)
2024-12-03T14:49:37,932 INFO  [M:0;a5d22df9eca2:34929 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-03T14:49:37,932 INFO  [M:0;a5d22df9eca2:34929 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T14:49:37,932 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:49:37,932 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:37,932 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:37,932 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:49:37,932 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:37,933 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-12-03T14:49:37,957 DEBUG [M:0;a5d22df9eca2:34929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac4d4dbc16d5482db01e64faabeec885 is 82, key is hbase:meta,,1/info:regioninfo/1733237284667/Put/seqid=0
2024-12-03T14:49:37,962 INFO  [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:49:37,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741855_1031 (size=5672)
2024-12-03T14:49:37,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741855_1031 (size=5672)
2024-12-03T14:49:37,963 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac4d4dbc16d5482db01e64faabeec885
2024-12-03T14:49:37,992 DEBUG [M:0;a5d22df9eca2:34929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6178450015c44fc495b53b189cfbff5b is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733237285570/Put/seqid=0
2024-12-03T14:49:37,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741856_1032 (size=6247)
2024-12-03T14:49:37,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741856_1032 (size=6247)
2024-12-03T14:49:37,998 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6178450015c44fc495b53b189cfbff5b
2024-12-03T14:49:38,004 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6178450015c44fc495b53b189cfbff5b
2024-12-03T14:49:38,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:38,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x100a08d6df10001, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:38,015 INFO  [RS:0;a5d22df9eca2:38513 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:49:38,016 INFO  [RS:0;a5d22df9eca2:38513 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,38513,1733237282685; zookeeper connection closed.
2024-12-03T14:49:38,016 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@139b7158 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@139b7158
2024-12-03T14:49:38,017 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T14:49:38,022 DEBUG [M:0;a5d22df9eca2:34929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b23dcc1cf954943a3304a1725892c9a is 69, key is a5d22df9eca2,38513,1733237282685/rs:state/1733237283856/Put/seqid=0
2024-12-03T14:49:38,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741857_1033 (size=5156)
2024-12-03T14:49:38,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741857_1033 (size=5156)
2024-12-03T14:49:38,029 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b23dcc1cf954943a3304a1725892c9a
2024-12-03T14:49:38,051 DEBUG [M:0;a5d22df9eca2:34929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b4959e6b929640e5b792d812edfaf0a7 is 52, key is load_balancer_on/state:d/1733237285039/Put/seqid=0
2024-12-03T14:49:38,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741858_1034 (size=5056)
2024-12-03T14:49:38,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741858_1034 (size=5056)
2024-12-03T14:49:38,060 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b4959e6b929640e5b792d812edfaf0a7
2024-12-03T14:49:38,068 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac4d4dbc16d5482db01e64faabeec885 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ac4d4dbc16d5482db01e64faabeec885
2024-12-03T14:49:38,074 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ac4d4dbc16d5482db01e64faabeec885, entries=8, sequenceid=59, filesize=5.5 K
2024-12-03T14:49:38,075 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6178450015c44fc495b53b189cfbff5b as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6178450015c44fc495b53b189cfbff5b
2024-12-03T14:49:38,082 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6178450015c44fc495b53b189cfbff5b
2024-12-03T14:49:38,082 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6178450015c44fc495b53b189cfbff5b, entries=6, sequenceid=59, filesize=6.1 K
2024-12-03T14:49:38,083 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b23dcc1cf954943a3304a1725892c9a as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6b23dcc1cf954943a3304a1725892c9a
2024-12-03T14:49:38,089 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6b23dcc1cf954943a3304a1725892c9a, entries=1, sequenceid=59, filesize=5.0 K
2024-12-03T14:49:38,091 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b4959e6b929640e5b792d812edfaf0a7 as hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b4959e6b929640e5b792d812edfaf0a7
2024-12-03T14:49:38,098 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b4959e6b929640e5b792d812edfaf0a7, entries=1, sequenceid=59, filesize=4.9 K
2024-12-03T14:49:38,099 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false
2024-12-03T14:49:38,101 INFO  [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:38,101 DEBUG [M:0;a5d22df9eca2:34929 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237377932Disabling compacts and flushes for region at 1733237377932Disabling writes for close at 1733237377932Obtaining lock to block concurrent updates at 1733237377933 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237377933Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1733237377933Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237377934 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237377934Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237377956 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237377956Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237377970 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237377991 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237377991Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237378004 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237378022 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237378022Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733237378036 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733237378050 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733237378050Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73fa60f1: reopening flushed file at 1733237378066 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f25d8e2: reopening flushed file at 1733237378074 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f90e2ac: reopening flushed file at 1733237378082 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a6ce2ab: reopening flushed file at 1733237378090 (+8 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false at 1733237378099 (+9 ms)Writing region close event to WAL at 1733237378101 (+2 ms)Closed at 1733237378101
2024-12-03T14:49:38,102 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:38,102 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:38,102 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:38,102 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:38,102 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:38,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38699 is added to blk_1073741830_1006 (size=27973)
2024-12-03T14:49:38,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741830_1006 (size=27973)
2024-12-03T14:49:38,106 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:49:38,106 INFO  [M:0;a5d22df9eca2:34929 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-03T14:49:38,106 INFO  [M:0;a5d22df9eca2:34929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34929
2024-12-03T14:49:38,106 INFO  [M:0;a5d22df9eca2:34929 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:49:38,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:38,208 INFO  [M:0;a5d22df9eca2:34929 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:49:38,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34929-0x100a08d6df10000, quorum=127.0.0.1:51937, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:38,217 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:38,220 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:38,220 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:38,220 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:38,220 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:38,223 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:38,223 WARN  [BP-598383475-172.17.0.2-1733237278962 heartbeating to localhost/127.0.0.1:43675 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:38,224 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:38,224 WARN  [BP-598383475-172.17.0.2-1733237278962 heartbeating to localhost/127.0.0.1:43675 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-598383475-172.17.0.2-1733237278962 (Datanode Uuid dff57559-aca9-4cb7-8321-df80f685e4cf) service to localhost/127.0.0.1:43675
2024-12-03T14:49:38,225 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data3/current/BP-598383475-172.17.0.2-1733237278962 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:38,225 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data4/current/BP-598383475-172.17.0.2-1733237278962 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:38,226 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:38,228 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:38,229 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:38,229 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:38,229 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:38,229 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:38,230 WARN  [BP-598383475-172.17.0.2-1733237278962 heartbeating to localhost/127.0.0.1:43675 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:38,230 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:38,230 WARN  [BP-598383475-172.17.0.2-1733237278962 heartbeating to localhost/127.0.0.1:43675 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-598383475-172.17.0.2-1733237278962 (Datanode Uuid a7e01857-2e5f-4b87-a2b4-7fda81e8d6ef) service to localhost/127.0.0.1:43675
2024-12-03T14:49:38,230 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:38,231 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data1/current/BP-598383475-172.17.0.2-1733237278962 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:38,231 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/cluster_277dfcb8-e65a-dfd3-9b84-0561555c1fbf/data/data2/current/BP-598383475-172.17.0.2-1733237278962 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:38,232 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:38,238 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:49:38,239 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:38,239 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:38,239 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:38,239 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:38,246 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-03T14:49:38,275 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-03T14:49:38,282 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43675
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:43675
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43675
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43675
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/a5d22df9eca2:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43675 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43675
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/a5d22df9eca2:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@6a73af6c
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43675 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43675 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/a5d22df9eca2:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 416), ProcessCount=11 (was 11), AvailableMemoryMB=5765 (was 6875)
2024-12-03T14:49:38,288 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=5764
2024-12-03T14:49:38,288 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T14:49:38,288 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.log.dir so I do NOT create it in target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342
2024-12-03T14:49:38,288 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b01db91-f0bf-180d-b39e-087d4b3e8572/hadoop.tmp.dir so I do NOT create it in target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede, deleteOnExit=true
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/test.cache.data in system properties and HBase conf
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir in system properties and HBase conf
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T14:49:38,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-03T14:49:38,289 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T14:49:38,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/nfs.dump.dir in system properties and HBase conf
2024-12-03T14:49:38,291 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/java.io.tmpdir in system properties and HBase conf
2024-12-03T14:49:38,291 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:49:38,291 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T14:49:38,291 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T14:49:38,303 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:49:38,354 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:38,361 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:38,362 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:38,362 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:38,362 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:49:38,363 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:38,364 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@534816f2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:38,364 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3bd9c5b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:38,457 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14a3b236{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/java.io.tmpdir/jetty-localhost-37985-hadoop-hdfs-3_4_1-tests_jar-_-any-14183671243828583659/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:49:38,458 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@630e1a46{HTTP/1.1, (http/1.1)}{localhost:37985}
2024-12-03T14:49:38,458 INFO  [Time-limited test {}] server.Server(415): Started @101765ms
2024-12-03T14:49:38,470 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:49:38,524 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:38,527 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:38,528 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:38,528 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:38,528 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:49:38,528 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68b9cf2c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:38,528 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55f7876e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:38,620 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2f841e9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/java.io.tmpdir/jetty-localhost-39191-hadoop-hdfs-3_4_1-tests_jar-_-any-4124131648645656782/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:38,620 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3270c9ae{HTTP/1.1, (http/1.1)}{localhost:39191}
2024-12-03T14:49:38,620 INFO  [Time-limited test {}] server.Server(415): Started @101927ms
2024-12-03T14:49:38,622 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:49:38,656 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:38,659 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:38,660 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:38,660 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:38,660 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:49:38,661 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4da0f787{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:38,661 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@463a48f5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:38,689 WARN  [Thread-436 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data1/current/BP-208340645-172.17.0.2-1733237378314/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:38,689 WARN  [Thread-437 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data2/current/BP-208340645-172.17.0.2-1733237378314/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:38,709 WARN  [Thread-415 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:49:38,712 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdf3986c3ae699189 with lease ID 0x5c73bc506733fc10: Processing first storage report for DS-70e22dee-5328-463c-bafe-ca088588e6c2 from datanode DatanodeRegistration(127.0.0.1:45505, datanodeUuid=8671bfe8-c841-47ee-96c7-58ed5caace13, infoPort=39077, infoSecurePort=0, ipcPort=40677, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314)
2024-12-03T14:49:38,712 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdf3986c3ae699189 with lease ID 0x5c73bc506733fc10: from storage DS-70e22dee-5328-463c-bafe-ca088588e6c2 node DatanodeRegistration(127.0.0.1:45505, datanodeUuid=8671bfe8-c841-47ee-96c7-58ed5caace13, infoPort=39077, infoSecurePort=0, ipcPort=40677, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:49:38,712 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdf3986c3ae699189 with lease ID 0x5c73bc506733fc10: Processing first storage report for DS-9c98aa79-72a9-477c-93d6-e2de211472ea from datanode DatanodeRegistration(127.0.0.1:45505, datanodeUuid=8671bfe8-c841-47ee-96c7-58ed5caace13, infoPort=39077, infoSecurePort=0, ipcPort=40677, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314)
2024-12-03T14:49:38,712 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdf3986c3ae699189 with lease ID 0x5c73bc506733fc10: from storage DS-9c98aa79-72a9-477c-93d6-e2de211472ea node DatanodeRegistration(127.0.0.1:45505, datanodeUuid=8671bfe8-c841-47ee-96c7-58ed5caace13, infoPort=39077, infoSecurePort=0, ipcPort=40677, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:38,763 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@788d8945{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/java.io.tmpdir/jetty-localhost-42807-hadoop-hdfs-3_4_1-tests_jar-_-any-18371127197001960188/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:38,764 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c30f553{HTTP/1.1, (http/1.1)}{localhost:42807}
2024-12-03T14:49:38,764 INFO  [Time-limited test {}] server.Server(415): Started @102070ms
2024-12-03T14:49:38,765 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:49:38,825 WARN  [Thread-462 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data3/current/BP-208340645-172.17.0.2-1733237378314/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:38,826 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data4/current/BP-208340645-172.17.0.2-1733237378314/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:38,840 WARN  [Thread-451 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:49:38,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x23b24013abd005e7 with lease ID 0x5c73bc506733fc11: Processing first storage report for DS-e61a43f0-ff26-4f25-9ada-5e41553cc7c2 from datanode DatanodeRegistration(127.0.0.1:39839, datanodeUuid=25c505d6-bc33-469c-bef9-3248c79ed2e9, infoPort=36933, infoSecurePort=0, ipcPort=35737, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314)
2024-12-03T14:49:38,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x23b24013abd005e7 with lease ID 0x5c73bc506733fc11: from storage DS-e61a43f0-ff26-4f25-9ada-5e41553cc7c2 node DatanodeRegistration(127.0.0.1:39839, datanodeUuid=25c505d6-bc33-469c-bef9-3248c79ed2e9, infoPort=36933, infoSecurePort=0, ipcPort=35737, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:49:38,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x23b24013abd005e7 with lease ID 0x5c73bc506733fc11: Processing first storage report for DS-0f47a45f-e881-464c-bbe5-e3a07276dacb from datanode DatanodeRegistration(127.0.0.1:39839, datanodeUuid=25c505d6-bc33-469c-bef9-3248c79ed2e9, infoPort=36933, infoSecurePort=0, ipcPort=35737, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314)
2024-12-03T14:49:38,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x23b24013abd005e7 with lease ID 0x5c73bc506733fc11: from storage DS-0f47a45f-e881-464c-bbe5-e3a07276dacb node DatanodeRegistration(127.0.0.1:39839, datanodeUuid=25c505d6-bc33-469c-bef9-3248c79ed2e9, infoPort=36933, infoSecurePort=0, ipcPort=35737, storageInfo=lv=-57;cid=testClusterID;nsid=1826350010;c=1733237378314), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:38,890 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342
2024-12-03T14:49:38,894 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/zookeeper_0, clientPort=49379, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T14:49:38,895 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49379
2024-12-03T14:49:38,896 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:38,898 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:38,915 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:49:38,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:49:38,917 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c with version=8
2024-12-03T14:49:38,917 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase-staging
2024-12-03T14:49:38,919 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:49:38,919 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:38,919 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:38,919 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:49:38,919 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:38,919 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:49:38,919 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-03T14:49:38,920 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:49:38,920 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42327
2024-12-03T14:49:38,921 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42327 connecting to ZooKeeper ensemble=127.0.0.1:49379
2024-12-03T14:49:38,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423270x0, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:49:38,926 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42327-0x100a08eebbb0000 connected
2024-12-03T14:49:38,950 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:38,951 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:38,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:38,954 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c, hbase.cluster.distributed=false
2024-12-03T14:49:38,956 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:49:38,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42327
2024-12-03T14:49:38,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42327
2024-12-03T14:49:38,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42327
2024-12-03T14:49:38,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42327
2024-12-03T14:49:38,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42327
2024-12-03T14:49:38,974 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:49:38,974 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:38,974 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:38,975 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:49:38,975 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:38,975 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:49:38,975 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:49:38,975 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:49:38,976 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38731
2024-12-03T14:49:38,977 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38731 connecting to ZooKeeper ensemble=127.0.0.1:49379
2024-12-03T14:49:38,978 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:38,980 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:38,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387310x0, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:49:38,985 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:38,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38731-0x100a08eebbb0001 connected
2024-12-03T14:49:38,986 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:49:38,986 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:49:38,987 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T14:49:38,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:49:38,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38731
2024-12-03T14:49:38,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38731
2024-12-03T14:49:38,993 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38731
2024-12-03T14:49:38,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38731
2024-12-03T14:49:38,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38731
2024-12-03T14:49:39,005 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:42327
2024-12-03T14:49:39,008 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:39,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:39,010 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T14:49:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,013 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:49:39,014 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,42327,1733237378919 from backup master directory
2024-12-03T14:49:39,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:39,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:39,015 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:49:39,015 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,021 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/hbase.id] with ID: fd386468-59e9-4119-83f0-23ba0fa69b7c
2024-12-03T14:49:39,021 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/.tmp/hbase.id
2024-12-03T14:49:39,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:49:39,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:49:39,035 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/.tmp/hbase.id]:[hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/hbase.id]
2024-12-03T14:49:39,052 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:39,052 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-03T14:49:39,054 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-03T14:49:39,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,064 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:49:39,065 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:49:39,066 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:49:39,067 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T14:49:39,067 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:49:39,079 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:49:39,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:49:39,081 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store
2024-12-03T14:49:39,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:49:39,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:49:39,089 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:39,089 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:49:39,089 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:39,089 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:39,089 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:49:39,089 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:39,090 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:39,090 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237379089Disabling compacts and flushes for region at 1733237379089Disabling writes for close at 1733237379089Writing region close event to WAL at 1733237379090 (+1 ms)Closed at 1733237379090
2024-12-03T14:49:39,091 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/.initializing
2024-12-03T14:49:39,092 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/WALs/a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,095 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C42327%2C1733237378919, suffix=, logDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/WALs/a5d22df9eca2,42327,1733237378919, archiveDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/oldWALs, maxLogs=10
2024-12-03T14:49:39,096 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C42327%2C1733237378919.1733237379096
2024-12-03T14:49:39,102 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/WALs/a5d22df9eca2,42327,1733237378919/a5d22df9eca2%2C42327%2C1733237378919.1733237379096
2024-12-03T14:49:39,103 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39077:39077),(127.0.0.1/127.0.0.1:36933:36933)]
2024-12-03T14:49:39,104 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:49:39,104 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:39,104 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,104 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,106 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:39,108 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T14:49:39,108 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,109 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,109 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:39,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T14:49:39,111 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:49:39,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:39,114 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T14:49:39,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:49:39,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:39,116 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T14:49:39,116 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,117 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:49:39,117 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,118 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,119 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,121 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,121 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,121 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T14:49:39,123 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:39,126 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:49:39,126 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=758783, jitterRate=-0.03515838086605072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T14:49:39,128 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733237379104Initializing all the Stores at 1733237379106 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237379106Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237379106Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237379106Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237379106Cleaning up temporary data from old regions at 1733237379121 (+15 ms)Region opened successfully at 1733237379127 (+6 ms)
2024-12-03T14:49:39,129 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T14:49:39,133 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2edd375b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:49:39,135 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-03T14:49:39,135 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T14:49:39,135 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T14:49:39,135 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T14:49:39,136 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T14:49:39,137 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-03T14:49:39,137 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T14:49:39,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T14:49:39,141 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T14:49:39,142 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-03T14:49:39,142 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T14:49:39,143 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T14:49:39,143 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-03T14:49:39,144 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T14:49:39,145 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T14:49:39,145 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-03T14:49:39,146 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T14:49:39,147 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T14:49:39,149 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T14:49:39,150 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T14:49:39,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:39,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:39,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,152 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,42327,1733237378919, sessionid=0x100a08eebbb0000, setting cluster-up flag (Was=false)
2024-12-03T14:49:39,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T14:49:39,158 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,163 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T14:49:39,164 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,166 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-03T14:49:39,167 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:39,168 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-03T14:49:39,168 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T14:49:39,168 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,42327,1733237378919
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:49:39,170 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,171 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237409171
2024-12-03T14:49:39,171 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T14:49:39,172 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T14:49:39,172 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T14:49:39,172 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T14:49:39,172 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T14:49:39,172 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T14:49:39,172 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:39,172 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-03T14:49:39,172 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,173 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T14:49:39,173 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T14:49:39,173 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T14:49:39,173 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,173 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T14:49:39,173 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T14:49:39,174 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T14:49:39,174 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237379174,5,FailOnTimeoutGroup]
2024-12-03T14:49:39,177 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237379174,5,FailOnTimeoutGroup]
2024-12-03T14:49:39,177 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,177 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T14:49:39,177 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,177 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:49:39,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:49:39,183 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-03T14:49:39,184 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c
2024-12-03T14:49:39,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:49:39,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:49:39,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:39,193 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:49:39,194 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:49:39,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,195 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,195 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:49:39,196 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(746): ClusterId : fd386468-59e9-4119-83f0-23ba0fa69b7c
2024-12-03T14:49:39,196 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:49:39,197 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:49:39,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,198 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:49:39,198 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:49:39,198 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,198 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:49:39,200 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:49:39,200 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:49:39,200 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,200 DEBUG [RS:0;a5d22df9eca2:38731 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1184e37f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:49:39,200 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,201 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:49:39,202 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:49:39,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,203 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:49:39,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740
2024-12-03T14:49:39,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740
2024-12-03T14:49:39,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:49:39,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:49:39,206 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:49:39,208 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:49:39,210 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:49:39,211 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=862596, jitterRate=0.09684765338897705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:49:39,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733237379191Initializing all the Stores at 1733237379192 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237379192Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237379193 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237379193Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237379193Cleaning up temporary data from old regions at 1733237379206 (+13 ms)Region opened successfully at 1733237379212 (+6 ms)
2024-12-03T14:49:39,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:49:39,212 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:49:39,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:49:39,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:49:39,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:49:39,213 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:49:39,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237379212Disabling compacts and flushes for region at 1733237379212Disabling writes for close at 1733237379213 (+1 ms)Writing region close event to WAL at 1733237379213Closed at 1733237379213
2024-12-03T14:49:39,215 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:39,215 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-03T14:49:39,215 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T14:49:39,217 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:49:39,219 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T14:49:39,219 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:38731
2024-12-03T14:49:39,219 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:49:39,219 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:49:39,219 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:49:39,220 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,42327,1733237378919 with port=38731, startcode=1733237378974
2024-12-03T14:49:39,220 DEBUG [RS:0;a5d22df9eca2:38731 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:49:39,223 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:49:39,224 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,224 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42327 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,227 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c
2024-12-03T14:49:39,227 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41215
2024-12-03T14:49:39,227 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:49:39,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:49:39,229 DEBUG [RS:0;a5d22df9eca2:38731 {}] zookeeper.ZKUtil(111): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,229 WARN  [RS:0;a5d22df9eca2:38731 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:49:39,229 INFO  [RS:0;a5d22df9eca2:38731 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:49:39,229 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/WALs/a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,229 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,38731,1733237378974]
2024-12-03T14:49:39,233 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:49:39,235 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:49:39,236 INFO  [RS:0;a5d22df9eca2:38731 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:49:39,236 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,237 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:49:39,238 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:49:39,238 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,238 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,238 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,238 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,238 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:49:39,239 DEBUG [RS:0;a5d22df9eca2:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:49:39,240 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,240 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,240 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,240 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,240 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,240 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,38731,1733237378974-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:49:39,255 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:49:39,255 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,38731,1733237378974-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,255 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,255 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.Replication(171): a5d22df9eca2,38731,1733237378974 started
2024-12-03T14:49:39,268 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,268 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,38731,1733237378974, RpcServer on a5d22df9eca2/172.17.0.2:38731, sessionid=0x100a08eebbb0001
2024-12-03T14:49:39,268 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:49:39,268 DEBUG [RS:0;a5d22df9eca2:38731 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,268 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,38731,1733237378974'
2024-12-03T14:49:39,268 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:49:39,269 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:49:39,269 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:49:39,269 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:49:39,270 DEBUG [RS:0;a5d22df9eca2:38731 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,270 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,38731,1733237378974'
2024-12-03T14:49:39,270 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:49:39,270 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:49:39,270 DEBUG [RS:0;a5d22df9eca2:38731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:49:39,270 INFO  [RS:0;a5d22df9eca2:38731 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:49:39,271 INFO  [RS:0;a5d22df9eca2:38731 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:49:39,369 WARN  [a5d22df9eca2:42327 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-03T14:49:39,373 INFO  [RS:0;a5d22df9eca2:38731 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C38731%2C1733237378974, suffix=, logDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/WALs/a5d22df9eca2,38731,1733237378974, archiveDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/oldWALs, maxLogs=32
2024-12-03T14:49:39,376 INFO  [RS:0;a5d22df9eca2:38731 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38731%2C1733237378974.1733237379375
2024-12-03T14:49:39,385 INFO  [RS:0;a5d22df9eca2:38731 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/WALs/a5d22df9eca2,38731,1733237378974/a5d22df9eca2%2C38731%2C1733237378974.1733237379375
2024-12-03T14:49:39,386 DEBUG [RS:0;a5d22df9eca2:38731 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39077:39077),(127.0.0.1/127.0.0.1:36933:36933)]
2024-12-03T14:49:39,619 DEBUG [a5d22df9eca2:42327 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T14:49:39,621 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,625 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,38731,1733237378974, state=OPENING
2024-12-03T14:49:39,628 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T14:49:39,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,633 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:49:39,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:39,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:39,634 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,38731,1733237378974}]
2024-12-03T14:49:39,789 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T14:49:39,796 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52505, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T14:49:39,804 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-03T14:49:39,804 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:49:39,807 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C38731%2C1733237378974.meta, suffix=.meta, logDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/WALs/a5d22df9eca2,38731,1733237378974, archiveDir=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/oldWALs, maxLogs=32
2024-12-03T14:49:39,810 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C38731%2C1733237378974.meta.1733237379810.meta
2024-12-03T14:49:39,819 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/WALs/a5d22df9eca2,38731,1733237378974/a5d22df9eca2%2C38731%2C1733237378974.meta.1733237379810.meta
2024-12-03T14:49:39,821 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36933:36933),(127.0.0.1/127.0.0.1:39077:39077)]
2024-12-03T14:49:39,822 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:49:39,822 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T14:49:39,822 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T14:49:39,822 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T14:49:39,823 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T14:49:39,823 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:39,823 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-03T14:49:39,823 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-03T14:49:39,825 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:49:39,826 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:49:39,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,826 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,827 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:49:39,828 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:49:39,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,829 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,829 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:49:39,830 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:49:39,830 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,830 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,831 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:49:39,832 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:49:39,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:39,832 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:39,832 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:49:39,833 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740
2024-12-03T14:49:39,835 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740
2024-12-03T14:49:39,836 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:49:39,836 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:49:39,837 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:49:39,838 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:49:39,839 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=748481, jitterRate=-0.04825817048549652}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:49:39,839 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-03T14:49:39,840 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733237379823Writing region info on filesystem at 1733237379823Initializing all the Stores at 1733237379824 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237379824Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237379824Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237379824Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237379824Cleaning up temporary data from old regions at 1733237379836 (+12 ms)Running coprocessor post-open hooks at 1733237379839 (+3 ms)Region opened successfully at 1733237379840 (+1 ms)
2024-12-03T14:49:39,841 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237379789
2024-12-03T14:49:39,844 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T14:49:39,844 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-03T14:49:39,845 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,846 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,38731,1733237378974, state=OPEN
2024-12-03T14:49:39,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:49:39,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:49:39,849 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:39,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:39,852 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T14:49:39,852 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,38731,1733237378974 in 216 msec
2024-12-03T14:49:39,855 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T14:49:39,855 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 637 msec
2024-12-03T14:49:39,856 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:39,856 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-03T14:49:39,858 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:49:39,858 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,38731,1733237378974, seqNum=-1]
2024-12-03T14:49:39,858 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:49:39,860 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:49:39,868 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 699 msec
2024-12-03T14:49:39,868 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237379868, completionTime=-1
2024-12-03T14:49:39,868 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T14:49:39,868 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-03T14:49:39,870 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-03T14:49:39,870 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237439870
2024-12-03T14:49:39,870 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237499870
2024-12-03T14:49:39,870 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-03T14:49:39,870 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42327,1733237378919-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,871 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42327,1733237378919-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,871 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42327,1733237378919-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,871 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:42327, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,871 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,871 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,873 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-03T14:49:39,875 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.860sec
2024-12-03T14:49:39,875 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T14:49:39,876 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T14:49:39,876 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T14:49:39,876 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T14:49:39,876 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T14:49:39,876 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42327,1733237378919-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:49:39,876 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42327,1733237378919-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T14:49:39,878 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-03T14:49:39,879 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T14:49:39,879 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42327,1733237378919-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:39,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781d38e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:49:39,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,42327,-1 for getting cluster id
2024-12-03T14:49:39,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-03T14:49:39,898 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fd386468-59e9-4119-83f0-23ba0fa69b7c'
2024-12-03T14:49:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-03T14:49:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fd386468-59e9-4119-83f0-23ba0fa69b7c"

2024-12-03T14:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b400f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,42327,-1]
2024-12-03T14:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-03T14:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:39,902 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-03T14:49:39,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab35378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:49:39,904 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:49:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,38731,1733237378974, seqNum=-1]
2024-12-03T14:49:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:49:39,908 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:49:39,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,911 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:39,914 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-03T14:49:39,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-03T14:49:39,915 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:49:39,915 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:49:39,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:39,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:39,915 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-03T14:49:39,915 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T14:49:39,915 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=325101689, stopped=false
2024-12-03T14:49:39,915 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,42327,1733237378919
2024-12-03T14:49:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:39,917 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:49:39,917 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:49:39,918 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:49:39,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:39,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:39,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:39,918 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,38731,1733237378974' *****
2024-12-03T14:49:39,918 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:49:39,918 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:49:39,919 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:38731.
2024-12-03T14:49:39,919 DEBUG [RS:0;a5d22df9eca2:38731 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:49:39,919 DEBUG [RS:0;a5d22df9eca2:38731 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:49:39,919 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-03T14:49:39,920 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-03T14:49:39,920 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-03T14:49:39,920 DEBUG [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-03T14:49:39,920 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:49:39,920 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:49:39,920 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:49:39,920 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:49:39,920 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:49:39,920 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-03T14:49:39,944 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740/.tmp/ns/687049dbcd7e4d249e656138518ca024 is 43, key is default/ns:d/1733237379860/Put/seqid=0
2024-12-03T14:49:39,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741835_1011 (size=5153)
2024-12-03T14:49:39,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741835_1011 (size=5153)
2024-12-03T14:49:39,951 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740/.tmp/ns/687049dbcd7e4d249e656138518ca024
2024-12-03T14:49:39,961 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740/.tmp/ns/687049dbcd7e4d249e656138518ca024 as hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740/ns/687049dbcd7e4d249e656138518ca024
2024-12-03T14:49:39,969 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740/ns/687049dbcd7e4d249e656138518ca024, entries=2, sequenceid=6, filesize=5.0 K
2024-12-03T14:49:39,970 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false
2024-12-03T14:49:39,975 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-03T14:49:39,976 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:49:39,976 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:49:39,976 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237379920Running coprocessor pre-close hooks at 1733237379920Disabling compacts and flushes for region at 1733237379920Disabling writes for close at 1733237379920Obtaining lock to block concurrent updates at 1733237379920Preparing flush snapshotting stores in 1588230740 at 1733237379920Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733237379921 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733237379921Flushing 1588230740/ns: creating writer at 1733237379921Flushing 1588230740/ns: appending metadata at 1733237379943 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733237379943Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45ef9944: reopening flushed file at 1733237379960 (+17 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false at 1733237379970 (+10 ms)Writing region close event to WAL at 1733237379971 (+1 ms)Running coprocessor post-close hooks at 1733237379976 (+5 ms)Closed at 1733237379976
2024-12-03T14:49:39,977 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T14:49:40,120 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,38731,1733237378974; all regions closed.
2024-12-03T14:49:40,121 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,121 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,122 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,122 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,122 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741834_1010 (size=1152)
2024-12-03T14:49:40,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741834_1010 (size=1152)
2024-12-03T14:49:40,132 DEBUG [RS:0;a5d22df9eca2:38731 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/oldWALs
2024-12-03T14:49:40,132 INFO  [RS:0;a5d22df9eca2:38731 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C38731%2C1733237378974.meta:.meta(num 1733237379810)
2024-12-03T14:49:40,132 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,132 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,132 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,133 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,133 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741833_1009 (size=93)
2024-12-03T14:49:40,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741833_1009 (size=93)
2024-12-03T14:49:40,136 DEBUG [RS:0;a5d22df9eca2:38731 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/oldWALs
2024-12-03T14:49:40,137 INFO  [RS:0;a5d22df9eca2:38731 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C38731%2C1733237378974:(num 1733237379375)
2024-12-03T14:49:40,137 DEBUG [RS:0;a5d22df9eca2:38731 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:40,137 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:49:40,137 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:49:40,137 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-03T14:49:40,137 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:49:40,137 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:49:40,137 INFO  [RS:0;a5d22df9eca2:38731 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38731
2024-12-03T14:49:40,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:49:40,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,38731,1733237378974
2024-12-03T14:49:40,138 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:49:40,139 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,38731,1733237378974]
2024-12-03T14:49:40,140 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,38731,1733237378974 already deleted, retry=false
2024-12-03T14:49:40,140 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,38731,1733237378974 expired; onlineServers=0
2024-12-03T14:49:40,140 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,42327,1733237378919' *****
2024-12-03T14:49:40,140 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T14:49:40,140 INFO  [M:0;a5d22df9eca2:42327 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:49:40,140 INFO  [M:0;a5d22df9eca2:42327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:49:40,140 DEBUG [M:0;a5d22df9eca2:42327 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T14:49:40,140 DEBUG [M:0;a5d22df9eca2:42327 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T14:49:40,140 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T14:49:40,140 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237379174 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237379174,5,FailOnTimeoutGroup]
2024-12-03T14:49:40,140 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237379174 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237379174,5,FailOnTimeoutGroup]
2024-12-03T14:49:40,141 INFO  [M:0;a5d22df9eca2:42327 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-03T14:49:40,141 INFO  [M:0;a5d22df9eca2:42327 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:49:40,141 DEBUG [M:0;a5d22df9eca2:42327 {}] master.HMaster(1795): Stopping service threads
2024-12-03T14:49:40,141 INFO  [M:0;a5d22df9eca2:42327 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T14:49:40,141 INFO  [M:0;a5d22df9eca2:42327 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:49:40,141 INFO  [M:0;a5d22df9eca2:42327 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T14:49:40,141 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T14:49:40,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T14:49:40,142 DEBUG [M:0;a5d22df9eca2:42327 {}] zookeeper.ZKUtil(347): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T14:49:40,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:40,142 WARN  [M:0;a5d22df9eca2:42327 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T14:49:40,142 INFO  [M:0;a5d22df9eca2:42327 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/.lastflushedseqids
2024-12-03T14:49:40,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741836_1012 (size=99)
2024-12-03T14:49:40,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741836_1012 (size=99)
2024-12-03T14:49:40,149 INFO  [M:0;a5d22df9eca2:42327 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-03T14:49:40,150 INFO  [M:0;a5d22df9eca2:42327 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T14:49:40,150 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:49:40,150 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:40,150 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:40,150 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:49:40,150 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:40,150 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-03T14:49:40,172 DEBUG [M:0;a5d22df9eca2:42327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c54c4364774b4cbe95a68486cda5aded is 82, key is hbase:meta,,1/info:regioninfo/1733237379845/Put/seqid=0
2024-12-03T14:49:40,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741837_1013 (size=5672)
2024-12-03T14:49:40,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741837_1013 (size=5672)
2024-12-03T14:49:40,178 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c54c4364774b4cbe95a68486cda5aded
2024-12-03T14:49:40,197 DEBUG [M:0;a5d22df9eca2:42327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/99352ea995c3428485c1304f4dd56b68 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733237379867/Put/seqid=0
2024-12-03T14:49:40,201 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741838_1014 (size=5275)
2024-12-03T14:49:40,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741838_1014 (size=5275)
2024-12-03T14:49:40,203 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/99352ea995c3428485c1304f4dd56b68
2024-12-03T14:49:40,224 DEBUG [M:0;a5d22df9eca2:42327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d632dd5543cd47188b3065a7eaf8e8b8 is 69, key is a5d22df9eca2,38731,1733237378974/rs:state/1733237379225/Put/seqid=0
2024-12-03T14:49:40,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741839_1015 (size=5156)
2024-12-03T14:49:40,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741839_1015 (size=5156)
2024-12-03T14:49:40,237 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d632dd5543cd47188b3065a7eaf8e8b8
2024-12-03T14:49:40,239 INFO  [RS:0;a5d22df9eca2:38731 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:49:40,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:40,240 INFO  [RS:0;a5d22df9eca2:38731 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,38731,1733237378974; zookeeper connection closed.
2024-12-03T14:49:40,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x100a08eebbb0001, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:40,245 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a983c25 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a983c25
2024-12-03T14:49:40,245 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T14:49:40,264 DEBUG [M:0;a5d22df9eca2:42327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8a37e0c42cae4dc5becea69cdb18a080 is 52, key is load_balancer_on/state:d/1733237379913/Put/seqid=0
2024-12-03T14:49:40,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741840_1016 (size=5056)
2024-12-03T14:49:40,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741840_1016 (size=5056)
2024-12-03T14:49:40,276 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8a37e0c42cae4dc5becea69cdb18a080
2024-12-03T14:49:40,285 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c54c4364774b4cbe95a68486cda5aded as hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c54c4364774b4cbe95a68486cda5aded
2024-12-03T14:49:40,292 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c54c4364774b4cbe95a68486cda5aded, entries=8, sequenceid=29, filesize=5.5 K
2024-12-03T14:49:40,294 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/99352ea995c3428485c1304f4dd56b68 as hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/99352ea995c3428485c1304f4dd56b68
2024-12-03T14:49:40,300 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/99352ea995c3428485c1304f4dd56b68, entries=3, sequenceid=29, filesize=5.2 K
2024-12-03T14:49:40,301 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d632dd5543cd47188b3065a7eaf8e8b8 as hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d632dd5543cd47188b3065a7eaf8e8b8
2024-12-03T14:49:40,309 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d632dd5543cd47188b3065a7eaf8e8b8, entries=1, sequenceid=29, filesize=5.0 K
2024-12-03T14:49:40,310 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8a37e0c42cae4dc5becea69cdb18a080 as hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8a37e0c42cae4dc5becea69cdb18a080
2024-12-03T14:49:40,318 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41215/user/jenkins/test-data/6560cbe4-61fe-4acd-5359-2f7d78065c9c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8a37e0c42cae4dc5becea69cdb18a080, entries=1, sequenceid=29, filesize=4.9 K
2024-12-03T14:49:40,319 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=29, compaction requested=false
2024-12-03T14:49:40,321 INFO  [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:40,321 DEBUG [M:0;a5d22df9eca2:42327 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237380150Disabling compacts and flushes for region at 1733237380150Disabling writes for close at 1733237380150Obtaining lock to block concurrent updates at 1733237380150Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237380150Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733237380151 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237380152 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237380152Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237380172 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237380172Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237380183 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237380196 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237380196Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237380208 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237380223 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237380223Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733237380245 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733237380263 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733237380264 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dfff2a9: reopening flushed file at 1733237380284 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aab6ee8: reopening flushed file at 1733237380293 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@411d115a: reopening flushed file at 1733237380300 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@756c48bc: reopening flushed file at 1733237380309 (+9 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=29, compaction requested=false at 1733237380319 (+10 ms)Writing region close event to WAL at 1733237380320 (+1 ms)Closed at 1733237380320
2024-12-03T14:49:40,321 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,321 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,321 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,321 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,322 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:40,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39839 is added to blk_1073741830_1006 (size=10311)
2024-12-03T14:49:40,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45505 is added to blk_1073741830_1006 (size=10311)
2024-12-03T14:49:40,325 INFO  [M:0;a5d22df9eca2:42327 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-03T14:49:40,325 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:49:40,325 INFO  [M:0;a5d22df9eca2:42327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42327
2024-12-03T14:49:40,325 INFO  [M:0;a5d22df9eca2:42327 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:49:40,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:40,426 INFO  [M:0;a5d22df9eca2:42327 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:49:40,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42327-0x100a08eebbb0000, quorum=127.0.0.1:49379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:49:40,431 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@788d8945{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:40,432 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c30f553{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:40,432 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:40,433 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@463a48f5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:40,433 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4da0f787{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:40,436 WARN  [BP-208340645-172.17.0.2-1733237378314 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:40,436 WARN  [BP-208340645-172.17.0.2-1733237378314 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-208340645-172.17.0.2-1733237378314 (Datanode Uuid 25c505d6-bc33-469c-bef9-3248c79ed2e9) service to localhost/127.0.0.1:41215
2024-12-03T14:49:40,436 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:40,436 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:40,437 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data3/current/BP-208340645-172.17.0.2-1733237378314 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:40,437 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data4/current/BP-208340645-172.17.0.2-1733237378314 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:40,437 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:40,440 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2f841e9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:40,440 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3270c9ae{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:40,440 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:40,441 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55f7876e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:40,441 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68b9cf2c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:40,442 WARN  [BP-208340645-172.17.0.2-1733237378314 heartbeating to localhost/127.0.0.1:41215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:40,442 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:40,442 WARN  [BP-208340645-172.17.0.2-1733237378314 heartbeating to localhost/127.0.0.1:41215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-208340645-172.17.0.2-1733237378314 (Datanode Uuid 8671bfe8-c841-47ee-96c7-58ed5caace13) service to localhost/127.0.0.1:41215
2024-12-03T14:49:40,442 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:40,443 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data1/current/BP-208340645-172.17.0.2-1733237378314 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:40,443 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/cluster_4bb235bc-4134-0ae5-e8a3-d5314f65bede/data/data2/current/BP-208340645-172.17.0.2-1733237378314 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:40,444 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:40,450 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14a3b236{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:49:40,451 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@630e1a46{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:40,451 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:40,451 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3bd9c5b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:40,452 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@534816f2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:40,457 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-03T14:49:40,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-03T14:49:40,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.log.dir so I do NOT create it in target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ebdc40d9-ee58-b611-42e4-aaf09b94a342/hadoop.tmp.dir so I do NOT create it in target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f, deleteOnExit=true
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/test.cache.data in system properties and HBase conf
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir in system properties and HBase conf
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T14:49:40,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-03T14:49:40,477 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:49:40,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:49:40,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T14:49:40,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/nfs.dump.dir in system properties and HBase conf
2024-12-03T14:49:40,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir in system properties and HBase conf
2024-12-03T14:49:40,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:49:40,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T14:49:40,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T14:49:40,490 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:49:40,549 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:40,554 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:40,557 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:40,557 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:40,557 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:49:40,558 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:40,559 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15d1211f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:40,560 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16369da1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:40,661 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4705e615{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir/jetty-localhost-35355-hadoop-hdfs-3_4_1-tests_jar-_-any-5777662372095986824/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:49:40,661 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d8a9c69{HTTP/1.1, (http/1.1)}{localhost:35355}
2024-12-03T14:49:40,661 INFO  [Time-limited test {}] server.Server(415): Started @103968ms
2024-12-03T14:49:40,673 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:49:40,723 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:40,726 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:40,727 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:40,727 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:40,727 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:49:40,727 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49b91397{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:40,728 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35a03ba9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:40,833 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@24befc55{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir/jetty-localhost-41075-hadoop-hdfs-3_4_1-tests_jar-_-any-16282832273115215333/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:40,833 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@259c861e{HTTP/1.1, (http/1.1)}{localhost:41075}
2024-12-03T14:49:40,833 INFO  [Time-limited test {}] server.Server(415): Started @104140ms
2024-12-03T14:49:40,835 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:49:40,868 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:40,871 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:40,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:40,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:40,872 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:49:40,873 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75096fee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:40,873 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d3d4ef0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:40,906 WARN  [Thread-655 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data1/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:40,909 WARN  [Thread-656 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data2/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:40,938 WARN  [Thread-634 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:49:40,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe5dec41138610204 with lease ID 0x16c1df3a3380a362: Processing first storage report for DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20 from datanode DatanodeRegistration(127.0.0.1:37547, datanodeUuid=fe42c28d-56fe-487f-a8fe-815b840a6db0, infoPort=43001, infoSecurePort=0, ipcPort=46555, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:40,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe5dec41138610204 with lease ID 0x16c1df3a3380a362: from storage DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20 node DatanodeRegistration(127.0.0.1:37547, datanodeUuid=fe42c28d-56fe-487f-a8fe-815b840a6db0, infoPort=43001, infoSecurePort=0, ipcPort=46555, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:49:40,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe5dec41138610204 with lease ID 0x16c1df3a3380a362: Processing first storage report for DS-3dd1469e-c7c1-414c-a0da-7bd19354caec from datanode DatanodeRegistration(127.0.0.1:37547, datanodeUuid=fe42c28d-56fe-487f-a8fe-815b840a6db0, infoPort=43001, infoSecurePort=0, ipcPort=46555, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:40,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe5dec41138610204 with lease ID 0x16c1df3a3380a362: from storage DS-3dd1469e-c7c1-414c-a0da-7bd19354caec node DatanodeRegistration(127.0.0.1:37547, datanodeUuid=fe42c28d-56fe-487f-a8fe-815b840a6db0, infoPort=43001, infoSecurePort=0, ipcPort=46555, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:41,005 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75de815b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir/jetty-localhost-43861-hadoop-hdfs-3_4_1-tests_jar-_-any-3172529665999531538/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:41,006 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@fe8f7b0{HTTP/1.1, (http/1.1)}{localhost:43861}
2024-12-03T14:49:41,006 INFO  [Time-limited test {}] server.Server(415): Started @104313ms
2024-12-03T14:49:41,008 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:49:41,076 WARN  [Thread-681 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data3/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:41,077 WARN  [Thread-682 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data4/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:41,097 WARN  [Thread-670 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:49:41,100 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1591daa58fd6e03c with lease ID 0x16c1df3a3380a363: Processing first storage report for DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c from datanode DatanodeRegistration(127.0.0.1:42953, datanodeUuid=eabbce3a-c616-4ab8-93bd-18ce1ba3c675, infoPort=42271, infoSecurePort=0, ipcPort=43815, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:41,100 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1591daa58fd6e03c with lease ID 0x16c1df3a3380a363: from storage DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c node DatanodeRegistration(127.0.0.1:42953, datanodeUuid=eabbce3a-c616-4ab8-93bd-18ce1ba3c675, infoPort=42271, infoSecurePort=0, ipcPort=43815, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:41,100 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1591daa58fd6e03c with lease ID 0x16c1df3a3380a363: Processing first storage report for DS-8466e49a-b879-42ce-abdc-a1a6471ed4d1 from datanode DatanodeRegistration(127.0.0.1:42953, datanodeUuid=eabbce3a-c616-4ab8-93bd-18ce1ba3c675, infoPort=42271, infoSecurePort=0, ipcPort=43815, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:41,100 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1591daa58fd6e03c with lease ID 0x16c1df3a3380a363: from storage DS-8466e49a-b879-42ce-abdc-a1a6471ed4d1 node DatanodeRegistration(127.0.0.1:42953, datanodeUuid=eabbce3a-c616-4ab8-93bd-18ce1ba3c675, infoPort=42271, infoSecurePort=0, ipcPort=43815, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:41,135 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397
2024-12-03T14:49:41,138 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/zookeeper_0, clientPort=63946, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T14:49:41,139 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63946
2024-12-03T14:49:41,139 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:41,141 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:41,154 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:49:41,154 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:49:41,155 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187 with version=8
2024-12-03T14:49:41,155 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase-staging
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-03T14:49:41,158 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:49:41,159 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37049
2024-12-03T14:49:41,160 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37049 connecting to ZooKeeper ensemble=127.0.0.1:63946
2024-12-03T14:49:41,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:370490x0, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:49:41,164 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37049-0x100a08ef4800000 connected
2024-12-03T14:49:41,175 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:41,176 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:41,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:41,179 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187, hbase.cluster.distributed=false
2024-12-03T14:49:41,180 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:49:41,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37049
2024-12-03T14:49:41,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37049
2024-12-03T14:49:41,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37049
2024-12-03T14:49:41,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37049
2024-12-03T14:49:41,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37049
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:49:41,196 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:49:41,197 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34883
2024-12-03T14:49:41,198 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34883 connecting to ZooKeeper ensemble=127.0.0.1:63946
2024-12-03T14:49:41,198 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:41,200 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:41,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348830x0, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:49:41,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34883-0x100a08ef4800001 connected
2024-12-03T14:49:41,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:49:41,205 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:49:41,209 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:49:41,209 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T14:49:41,211 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:49:41,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34883
2024-12-03T14:49:41,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34883
2024-12-03T14:49:41,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34883
2024-12-03T14:49:41,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34883
2024-12-03T14:49:41,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34883
2024-12-03T14:49:41,227 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:37049
2024-12-03T14:49:41,227 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:41,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:41,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:41,229 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:41,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T14:49:41,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,230 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:49:41,231 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,37049,1733237381157 from backup master directory
2024-12-03T14:49:41,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:41,231 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:49:41,231 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:41,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:41,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:49:41,236 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/hbase.id] with ID: e64e011a-8fa2-4f94-a40b-032ee2cdc672
2024-12-03T14:49:41,236 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/.tmp/hbase.id
2024-12-03T14:49:41,241 INFO  [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:49:41,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:49:41,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:49:41,248 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/.tmp/hbase.id]:[hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/hbase.id]
2024-12-03T14:49:41,261 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:41,261 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-03T14:49:41,263 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-03T14:49:41,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:49:41,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:49:41,683 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:49:41,685 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T14:49:41,685 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:49:41,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:49:41,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:49:41,698 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store
2024-12-03T14:49:41,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:49:41,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:49:41,710 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:41,710 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:49:41,710 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:41,710 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:41,710 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:49:41,710 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:41,710 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:49:41,711 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237381710Disabling compacts and flushes for region at 1733237381710Disabling writes for close at 1733237381710Writing region close event to WAL at 1733237381710Closed at 1733237381710
2024-12-03T14:49:41,712 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/.initializing
2024-12-03T14:49:41,712 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:41,715 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C37049%2C1733237381157, suffix=, logDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157, archiveDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/oldWALs, maxLogs=10
2024-12-03T14:49:41,716 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C37049%2C1733237381157.1733237381716
2024-12-03T14:49:41,722 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
2024-12-03T14:49:41,729 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43001:43001),(127.0.0.1/127.0.0.1:42271:42271)]
2024-12-03T14:49:41,739 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:49:41,739 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:41,740 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,740 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,743 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:41,745 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T14:49:41,745 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,745 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:41,746 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:41,747 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T14:49:41,747 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,748 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:49:41,749 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:41,750 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T14:49:41,751 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,751 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:49:41,751 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:49:41,753 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T14:49:41,753 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,754 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:49:41,754 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,755 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,756 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,758 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,758 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,758 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T14:49:41,760 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:49:41,762 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:49:41,764 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=860496, jitterRate=0.09417776763439178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T14:49:41,765 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733237381740Initializing all the Stores at 1733237381742 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237381742Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237381742Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237381742Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237381742Cleaning up temporary data from old regions at 1733237381758 (+16 ms)Region opened successfully at 1733237381765 (+7 ms)
2024-12-03T14:49:41,765 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T14:49:41,770 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77098c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:49:41,771 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-03T14:49:41,772 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T14:49:41,772 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T14:49:41,772 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T14:49:41,773 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T14:49:41,773 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-03T14:49:41,773 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T14:49:41,779 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T14:49:41,780 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T14:49:41,781 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-03T14:49:41,781 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T14:49:41,782 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T14:49:41,783 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-03T14:49:41,783 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T14:49:41,784 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T14:49:41,785 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-03T14:49:41,786 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T14:49:41,787 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T14:49:41,789 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T14:49:41,790 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T14:49:41,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:41,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:49:41,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,792 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,37049,1733237381157, sessionid=0x100a08ef4800000, setting cluster-up flag (Was=false)
2024-12-03T14:49:41,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,797 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T14:49:41,798 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:41,804 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T14:49:41,805 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:41,810 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-03T14:49:41,812 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:41,812 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-03T14:49:41,812 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T14:49:41,812 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,37049,1733237381157
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:49:41,814 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,818 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237411818
2024-12-03T14:49:41,818 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T14:49:41,818 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T14:49:41,818 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T14:49:41,818 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T14:49:41,819 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(746): ClusterId : e64e011a-8fa2-4f94-a40b-032ee2cdc672
2024-12-03T14:49:41,819 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:49:41,818 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T14:49:41,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T14:49:41,819 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:41,819 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-03T14:49:41,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,820 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T14:49:41,820 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T14:49:41,820 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T14:49:41,821 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T14:49:41,822 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:49:41,822 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T14:49:41,822 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:49:41,822 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237381822,5,FailOnTimeoutGroup]
2024-12-03T14:49:41,824 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:49:41,829 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237381822,5,FailOnTimeoutGroup]
2024-12-03T14:49:41,829 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,829 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T14:49:41,829 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,829 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,833 DEBUG [RS:0;a5d22df9eca2:34883 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e8bf12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:49:41,833 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,833 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T14:49:41,854 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:34883
2024-12-03T14:49:41,854 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:49:41,854 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:49:41,854 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:49:41,858 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,37049,1733237381157 with port=34883, startcode=1733237381195
2024-12-03T14:49:41,858 DEBUG [RS:0;a5d22df9eca2:34883 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:49:41,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:49:41,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:49:41,861 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-03T14:49:41,862 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187
2024-12-03T14:49:41,862 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:49:41,862 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37049 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:41,863 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37049 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:41,866 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187
2024-12-03T14:49:41,867 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35825
2024-12-03T14:49:41,867 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:49:41,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:49:41,871 DEBUG [RS:0;a5d22df9eca2:34883 {}] zookeeper.ZKUtil(111): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:41,871 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,34883,1733237381195]
2024-12-03T14:49:41,871 WARN  [RS:0;a5d22df9eca2:34883 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:49:41,872 INFO  [RS:0;a5d22df9eca2:34883 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:49:41,872 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:41,890 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:49:41,897 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:49:41,898 INFO  [RS:0;a5d22df9eca2:34883 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:49:41,898 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,899 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:49:41,901 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:49:41,901 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,901 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,902 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,903 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,903 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:49:41,904 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:41,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:49:41,904 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:49:41,904 DEBUG [RS:0;a5d22df9eca2:34883 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:49:41,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:41,913 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,913 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,913 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,914 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,914 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,914 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34883,1733237381195-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:49:41,919 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:49:41,922 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:49:41,922 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,923 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:41,923 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:49:41,926 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:49:41,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,927 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:41,927 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:49:41,929 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:49:41,929 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,930 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:41,930 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:49:41,932 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:49:41,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:41,933 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:41,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:49:41,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740
2024-12-03T14:49:41,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740
2024-12-03T14:49:41,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:49:41,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:49:41,940 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:49:41,942 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:49:41,942 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34883,1733237381195-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,943 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,943 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.Replication(171): a5d22df9eca2,34883,1733237381195 started
2024-12-03T14:49:41,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:49:41,950 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:49:41,951 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=766344, jitterRate=-0.02554328739643097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:49:41,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733237381905Initializing all the Stores at 1733237381907 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237381907Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237381919 (+12 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237381919Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237381919Cleaning up temporary data from old regions at 1733237381939 (+20 ms)Region opened successfully at 1733237381952 (+13 ms)
2024-12-03T14:49:41,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:49:41,952 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:49:41,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:49:41,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:49:41,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:49:41,958 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:49:41,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237381952Disabling compacts and flushes for region at 1733237381952Disabling writes for close at 1733237381953 (+1 ms)Writing region close event to WAL at 1733237381958 (+5 ms)Closed at 1733237381958
2024-12-03T14:49:41,960 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:41,960 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-03T14:49:41,960 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T14:49:41,960 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:41,960 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,34883,1733237381195, RpcServer on a5d22df9eca2/172.17.0.2:34883, sessionid=0x100a08ef4800001
2024-12-03T14:49:41,960 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:49:41,960 DEBUG [RS:0;a5d22df9eca2:34883 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:41,960 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,34883,1733237381195'
2024-12-03T14:49:41,961 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:49:41,961 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:49:41,962 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:49:41,962 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:49:41,962 DEBUG [RS:0;a5d22df9eca2:34883 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:41,962 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,34883,1733237381195'
2024-12-03T14:49:41,962 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:49:41,963 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:49:41,964 DEBUG [RS:0;a5d22df9eca2:34883 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:49:41,964 INFO  [RS:0;a5d22df9eca2:34883 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:49:41,964 INFO  [RS:0;a5d22df9eca2:34883 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:49:41,965 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:49:41,967 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T14:49:42,067 INFO  [RS:0;a5d22df9eca2:34883 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C34883%2C1733237381195, suffix=, logDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195, archiveDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs, maxLogs=32
2024-12-03T14:49:42,068 INFO  [RS:0;a5d22df9eca2:34883 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.1733237382068
2024-12-03T14:49:42,077 INFO  [RS:0;a5d22df9eca2:34883 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068
2024-12-03T14:49:42,085 DEBUG [RS:0;a5d22df9eca2:34883 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42271:42271),(127.0.0.1/127.0.0.1:43001:43001)]
2024-12-03T14:49:42,118 DEBUG [a5d22df9eca2:37049 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T14:49:42,118 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:42,120 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,34883,1733237381195, state=OPENING
2024-12-03T14:49:42,121 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T14:49:42,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:42,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:49:42,123 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:49:42,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:42,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:42,123 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,34883,1733237381195}]
2024-12-03T14:49:42,277 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T14:49:42,280 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T14:49:42,285 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-03T14:49:42,286 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:49:42,288 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C34883%2C1733237381195.meta, suffix=.meta, logDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195, archiveDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs, maxLogs=32
2024-12-03T14:49:42,289 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
2024-12-03T14:49:42,296 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
2024-12-03T14:49:42,298 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43001:43001),(127.0.0.1/127.0.0.1:42271:42271)]
2024-12-03T14:49:42,299 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:49:42,299 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T14:49:42,299 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T14:49:42,299 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T14:49:42,300 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T14:49:42,300 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:42,300 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-03T14:49:42,300 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-03T14:49:42,302 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:49:42,303 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:49:42,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:42,304 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:42,304 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:49:42,305 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:49:42,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:42,306 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:42,307 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:49:42,308 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:49:42,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:42,308 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:42,309 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:49:42,310 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:49:42,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:42,311 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:49:42,311 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:49:42,312 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740
2024-12-03T14:49:42,314 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740
2024-12-03T14:49:42,315 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:49:42,315 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:49:42,316 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:49:42,317 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:49:42,318 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874722, jitterRate=0.11226682364940643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:49:42,318 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-03T14:49:42,319 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733237382300Writing region info on filesystem at 1733237382300Initializing all the Stores at 1733237382301 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237382301Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237382302 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237382302Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237382302Cleaning up temporary data from old regions at 1733237382315 (+13 ms)Running coprocessor post-open hooks at 1733237382318 (+3 ms)Region opened successfully at 1733237382319 (+1 ms)
2024-12-03T14:49:42,320 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237382277
2024-12-03T14:49:42,323 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T14:49:42,323 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-03T14:49:42,324 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:42,326 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,34883,1733237381195, state=OPEN
2024-12-03T14:49:42,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:49:42,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:49:42,328 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:42,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:42,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:49:42,332 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T14:49:42,332 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,34883,1733237381195 in 205 msec
2024-12-03T14:49:42,335 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T14:49:42,335 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 372 msec
2024-12-03T14:49:42,336 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:49:42,336 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-03T14:49:42,338 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:49:42,338 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,34883,1733237381195, seqNum=-1]
2024-12-03T14:49:42,338 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:49:42,340 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43167, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:49:42,347 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 534 msec
2024-12-03T14:49:42,347 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237382347, completionTime=-1
2024-12-03T14:49:42,347 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T14:49:42,347 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-03T14:49:42,349 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-03T14:49:42,349 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237442349
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237502350
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,37049,1733237381157-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,37049,1733237381157-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,37049,1733237381157-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:37049, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,350 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,352 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-03T14:49:42,354 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.122sec
2024-12-03T14:49:42,354 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T14:49:42,354 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T14:49:42,354 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T14:49:42,354 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T14:49:42,354 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T14:49:42,354 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,37049,1733237381157-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:49:42,355 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,37049,1733237381157-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T14:49:42,357 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-03T14:49:42,357 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T14:49:42,357 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,37049,1733237381157-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-03T14:49:42,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b2e9f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:49:42,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,37049,-1 for getting cluster id
2024-12-03T14:49:42,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-03T14:49:42,423 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e64e011a-8fa2-4f94-a40b-032ee2cdc672'
2024-12-03T14:49:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-03T14:49:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e64e011a-8fa2-4f94-a40b-032ee2cdc672"

2024-12-03T14:49:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d239e3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:49:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,37049,-1]
2024-12-03T14:49:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-03T14:49:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:49:42,429 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-03T14:49:42,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8113bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:49:42,430 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:49:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,34883,1733237381195, seqNum=-1]
2024-12-03T14:49:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:49:42,434 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:49:42,436 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:42,436 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:42,439 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-03T14:49:42,458 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:49:42,458 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:42,458 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:42,458 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:49:42,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:49:42,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:49:42,459 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:49:42,459 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:49:42,459 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44475
2024-12-03T14:49:42,460 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44475 connecting to ZooKeeper ensemble=127.0.0.1:63946
2024-12-03T14:49:42,461 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:42,462 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:49:42,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444750x0, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:49:42,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:444750x0, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-03T14:49:42,466 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-03T14:49:42,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44475-0x100a08ef4800002 connected
2024-12-03T14:49:42,467 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:49:42,467 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:49:42,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:49:42,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:49:42,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44475
2024-12-03T14:49:42,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44475
2024-12-03T14:49:42,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44475
2024-12-03T14:49:42,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44475
2024-12-03T14:49:42,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44475
2024-12-03T14:49:42,478 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(746): ClusterId : e64e011a-8fa2-4f94-a40b-032ee2cdc672
2024-12-03T14:49:42,478 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:49:42,480 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:49:42,480 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:49:42,481 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:49:42,482 DEBUG [RS:1;a5d22df9eca2:44475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a098ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:49:42,492 DEBUG [RS:1;a5d22df9eca2:44475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a5d22df9eca2:44475
2024-12-03T14:49:42,492 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:49:42,492 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:49:42,492 DEBUG [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:49:42,493 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,37049,1733237381157 with port=44475, startcode=1733237382458
2024-12-03T14:49:42,493 DEBUG [RS:1;a5d22df9eca2:44475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:49:42,495 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51239, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:49:42,495 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37049 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,44475,1733237382458
2024-12-03T14:49:42,495 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37049 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,44475,1733237382458
2024-12-03T14:49:42,497 DEBUG [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187
2024-12-03T14:49:42,497 DEBUG [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35825
2024-12-03T14:49:42,497 DEBUG [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:49:42,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:49:42,499 DEBUG [RS:1;a5d22df9eca2:44475 {}] zookeeper.ZKUtil(111): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,44475,1733237382458
2024-12-03T14:49:42,499 WARN  [RS:1;a5d22df9eca2:44475 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:49:42,499 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,44475,1733237382458]
2024-12-03T14:49:42,499 INFO  [RS:1;a5d22df9eca2:44475 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:49:42,499 DEBUG [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458
2024-12-03T14:49:42,503 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:49:42,504 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:49:42,505 INFO  [RS:1;a5d22df9eca2:44475 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:49:42,505 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,505 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:49:42,506 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:49:42,506 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,506 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,506 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,506 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,506 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,506 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:49:42,507 DEBUG [RS:1;a5d22df9eca2:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:49:42,507 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,508 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,508 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,508 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,508 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,508 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44475,1733237382458-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:49:42,527 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:49:42,527 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44475,1733237382458-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,528 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,528 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.Replication(171): a5d22df9eca2,44475,1733237382458 started
2024-12-03T14:49:42,543 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:49:42,543 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,44475,1733237382458, RpcServer on a5d22df9eca2/172.17.0.2:44475, sessionid=0x100a08ef4800002
2024-12-03T14:49:42,543 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:49:42,543 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;a5d22df9eca2:44475,5,FailOnTimeoutGroup]
2024-12-03T14:49:42,543 DEBUG [RS:1;a5d22df9eca2:44475 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,44475,1733237382458
2024-12-03T14:49:42,543 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,44475,1733237382458'
2024-12-03T14:49:42,543 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:49:42,544 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-03T14:49:42,544 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-03T14:49:42,544 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:49:42,545 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:49:42,545 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:49:42,545 DEBUG [RS:1;a5d22df9eca2:44475 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,44475,1733237382458
2024-12-03T14:49:42,545 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,44475,1733237382458'
2024-12-03T14:49:42,545 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:49:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is a5d22df9eca2,37049,1733237381157
2024-12-03T14:49:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b749963
2024-12-03T14:49:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T14:49:42,545 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:49:42,546 DEBUG [RS:1;a5d22df9eca2:44475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:49:42,546 INFO  [RS:1;a5d22df9eca2:44475 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:49:42,546 INFO  [RS:1;a5d22df9eca2:44475 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:49:42,547 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T14:49:42,547 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37049 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T14:49:42,547 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T14:49:42,548 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:49:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37049 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-03T14:49:42,550 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T14:49:42,551 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:42,551 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-03T14:49:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:49:42,552 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T14:49:42,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741835_1011 (size=393)
2024-12-03T14:49:42,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741835_1011 (size=393)
2024-12-03T14:49:42,560 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a67ed774ca7dff4fd79fd8580a3231fa, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187
2024-12-03T14:49:42,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42953 is added to blk_1073741836_1012 (size=76)
2024-12-03T14:49:42,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37547 is added to blk_1073741836_1012 (size=76)
2024-12-03T14:49:42,568 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:42,569 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing a67ed774ca7dff4fd79fd8580a3231fa, disabling compactions & flushes
2024-12-03T14:49:42,569 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:42,569 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:42,569 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa. after waiting 0 ms
2024-12-03T14:49:42,569 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:42,569 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:42,569 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for a67ed774ca7dff4fd79fd8580a3231fa:
Waiting for close lock at 1733237382569Disabling compacts and flushes for region at 1733237382569Disabling writes for close at 1733237382569Writing region close event to WAL at 1733237382569Closed at 1733237382569
2024-12-03T14:49:42,571 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T14:49:42,571 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733237382571"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237382571"}]},"ts":"1733237382571"}
2024-12-03T14:49:42,574 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-03T14:49:42,575 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T14:49:42,576 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237382575"}]},"ts":"1733237382575"}
2024-12-03T14:49:42,578 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-03T14:49:42,578 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a67ed774ca7dff4fd79fd8580a3231fa, ASSIGN}]
2024-12-03T14:49:42,580 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a67ed774ca7dff4fd79fd8580a3231fa, ASSIGN
2024-12-03T14:49:42,581 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a67ed774ca7dff4fd79fd8580a3231fa, ASSIGN; state=OFFLINE, location=a5d22df9eca2,34883,1733237381195; forceNewPlan=false, retain=false
2024-12-03T14:49:42,648 INFO  [RS:1;a5d22df9eca2:44475 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C44475%2C1733237382458, suffix=, logDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458, archiveDir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs, maxLogs=32
2024-12-03T14:49:42,649 INFO  [RS:1;a5d22df9eca2:44475 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C44475%2C1733237382458.1733237382649
2024-12-03T14:49:42,662 INFO  [RS:1;a5d22df9eca2:44475 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
2024-12-03T14:49:42,666 DEBUG [RS:1;a5d22df9eca2:44475 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43001:43001),(127.0.0.1/127.0.0.1:42271:42271)]
2024-12-03T14:49:42,732 INFO  [a5d22df9eca2:37049 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-03T14:49:42,732 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a67ed774ca7dff4fd79fd8580a3231fa, regionState=OPENING, regionLocation=a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:42,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a67ed774ca7dff4fd79fd8580a3231fa, ASSIGN because future has completed
2024-12-03T14:49:42,736 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a67ed774ca7dff4fd79fd8580a3231fa, server=a5d22df9eca2,34883,1733237381195}]
2024-12-03T14:49:42,757 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:42,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:42,894 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:42,894 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a67ed774ca7dff4fd79fd8580a3231fa, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:49:42,895 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,895 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:49:42,895 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,895 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,896 INFO  [StoreOpener-a67ed774ca7dff4fd79fd8580a3231fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a67ed774ca7dff4fd79fd8580a3231fa 
2024-12-03T14:49:42,898 INFO  [StoreOpener-a67ed774ca7dff4fd79fd8580a3231fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a67ed774ca7dff4fd79fd8580a3231fa columnFamilyName info
2024-12-03T14:49:42,898 DEBUG [StoreOpener-a67ed774ca7dff4fd79fd8580a3231fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:49:42,898 INFO  [StoreOpener-a67ed774ca7dff4fd79fd8580a3231fa-1 {}] regionserver.HStore(327): Store=a67ed774ca7dff4fd79fd8580a3231fa/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:49:42,898 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,899 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,900 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,900 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,900 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,902 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,904 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:49:42,905 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a67ed774ca7dff4fd79fd8580a3231fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=792867, jitterRate=0.008183300495147705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T14:49:42,905 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:42,906 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a67ed774ca7dff4fd79fd8580a3231fa:
Running coprocessor pre-open hook at 1733237382895Writing region info on filesystem at 1733237382895Initializing all the Stores at 1733237382896 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237382896Cleaning up temporary data from old regions at 1733237382900 (+4 ms)Running coprocessor post-open hooks at 1733237382905 (+5 ms)Region opened successfully at 1733237382906 (+1 ms)
2024-12-03T14:49:42,907 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa., pid=6, masterSystemTime=1733237382890
2024-12-03T14:49:42,910 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:42,910 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:42,910 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a67ed774ca7dff4fd79fd8580a3231fa, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,34883,1733237381195
2024-12-03T14:49:42,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a67ed774ca7dff4fd79fd8580a3231fa, server=a5d22df9eca2,34883,1733237381195 because future has completed
2024-12-03T14:49:42,918 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T14:49:42,919 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a67ed774ca7dff4fd79fd8580a3231fa, server=a5d22df9eca2,34883,1733237381195 in 179 msec
2024-12-03T14:49:42,921 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T14:49:42,921 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=a67ed774ca7dff4fd79fd8580a3231fa, ASSIGN in 340 msec
2024-12-03T14:49:42,923 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T14:49:42,923 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237382923"}]},"ts":"1733237382923"}
2024-12-03T14:49:42,925 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-03T14:49:42,926 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T14:49:42,929 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 378 msec
2024-12-03T14:49:43,277 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:49:43,279 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:43,303 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:43,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:43,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:47,890 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T14:49:47,891 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-03T14:49:48,400 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:49:48,404 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:48,433 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:48,436 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:48,436 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:49:52,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-03T14:49:52,366 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-03T14:49:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:49:52,626 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-03T14:49:52,626 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-03T14:49:52,633 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-03T14:49:52,633 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:49:52,651 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:52,654 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:52,655 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:52,655 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:52,655 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:49:52,656 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a38f74a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:52,656 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b53e4f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:52,747 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2488e9f7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir/jetty-localhost-39949-hadoop-hdfs-3_4_1-tests_jar-_-any-11864104449747091235/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:52,747 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@132d95f4{HTTP/1.1, (http/1.1)}{localhost:39949}
2024-12-03T14:49:52,747 INFO  [Time-limited test {}] server.Server(415): Started @116054ms
2024-12-03T14:49:52,749 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:49:52,778 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:52,780 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:52,781 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:52,781 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:52,781 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:49:52,781 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@547f223f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:52,781 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ca4b7c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:52,804 WARN  [Thread-828 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data6/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:52,804 WARN  [Thread-827 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data5/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:52,822 WARN  [Thread-807 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:49:52,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe400cf41c7a1a831 with lease ID 0x16c1df3a3380a364: Processing first storage report for DS-be8583c9-8a47-4ad5-b311-99bd4c422f49 from datanode DatanodeRegistration(127.0.0.1:40629, datanodeUuid=8974d62f-a711-4f42-8566-d2331a5ea5bc, infoPort=37891, infoSecurePort=0, ipcPort=36441, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:52,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe400cf41c7a1a831 with lease ID 0x16c1df3a3380a364: from storage DS-be8583c9-8a47-4ad5-b311-99bd4c422f49 node DatanodeRegistration(127.0.0.1:40629, datanodeUuid=8974d62f-a711-4f42-8566-d2331a5ea5bc, infoPort=37891, infoSecurePort=0, ipcPort=36441, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:49:52,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe400cf41c7a1a831 with lease ID 0x16c1df3a3380a364: Processing first storage report for DS-72cc5b74-c011-4e19-b033-e43c792a1a7e from datanode DatanodeRegistration(127.0.0.1:40629, datanodeUuid=8974d62f-a711-4f42-8566-d2331a5ea5bc, infoPort=37891, infoSecurePort=0, ipcPort=36441, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:52,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe400cf41c7a1a831 with lease ID 0x16c1df3a3380a364: from storage DS-72cc5b74-c011-4e19-b033-e43c792a1a7e node DatanodeRegistration(127.0.0.1:40629, datanodeUuid=8974d62f-a711-4f42-8566-d2331a5ea5bc, infoPort=37891, infoSecurePort=0, ipcPort=36441, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:52,874 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5f24e79d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir/jetty-localhost-35971-hadoop-hdfs-3_4_1-tests_jar-_-any-3574145128184041875/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:52,875 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3986ff43{HTTP/1.1, (http/1.1)}{localhost:35971}
2024-12-03T14:49:52,875 INFO  [Time-limited test {}] server.Server(415): Started @116181ms
2024-12-03T14:49:52,876 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:49:52,906 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:49:52,910 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:49:52,912 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:49:52,912 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:49:52,912 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:49:52,913 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ab1ed71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:49:52,913 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c141b19{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:49:52,933 WARN  [Thread-862 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data7/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:52,933 WARN  [Thread-863 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data8/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:52,951 WARN  [Thread-842 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:49:52,953 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7369e27c3bb24170 with lease ID 0x16c1df3a3380a365: Processing first storage report for DS-86c1a948-a62a-447b-8f80-428cc55823dc from datanode DatanodeRegistration(127.0.0.1:39769, datanodeUuid=1dd964a6-6aaa-4e71-9d31-7046659ed1df, infoPort=35509, infoSecurePort=0, ipcPort=37011, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:52,953 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7369e27c3bb24170 with lease ID 0x16c1df3a3380a365: from storage DS-86c1a948-a62a-447b-8f80-428cc55823dc node DatanodeRegistration(127.0.0.1:39769, datanodeUuid=1dd964a6-6aaa-4e71-9d31-7046659ed1df, infoPort=35509, infoSecurePort=0, ipcPort=37011, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:52,953 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7369e27c3bb24170 with lease ID 0x16c1df3a3380a365: Processing first storage report for DS-30d69960-cd0e-4d66-807a-a73a2034b01f from datanode DatanodeRegistration(127.0.0.1:39769, datanodeUuid=1dd964a6-6aaa-4e71-9d31-7046659ed1df, infoPort=35509, infoSecurePort=0, ipcPort=37011, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:52,953 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7369e27c3bb24170 with lease ID 0x16c1df3a3380a365: from storage DS-30d69960-cd0e-4d66-807a-a73a2034b01f node DatanodeRegistration(127.0.0.1:39769, datanodeUuid=1dd964a6-6aaa-4e71-9d31-7046659ed1df, infoPort=35509, infoSecurePort=0, ipcPort=37011, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:53,006 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@325d7ee2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir/jetty-localhost-36459-hadoop-hdfs-3_4_1-tests_jar-_-any-12640890916336592398/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:53,006 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15751333{HTTP/1.1, (http/1.1)}{localhost:36459}
2024-12-03T14:49:53,006 INFO  [Time-limited test {}] server.Server(415): Started @116313ms
2024-12-03T14:49:53,007 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:49:53,067 WARN  [Thread-888 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:53,067 WARN  [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10/current/BP-133690028-172.17.0.2-1733237380503/current, will proceed with Du for space computation calculation, 
2024-12-03T14:49:53,084 WARN  [Thread-877 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:49:53,086 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3ed988a340607d50 with lease ID 0x16c1df3a3380a366: Processing first storage report for DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319 from datanode DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:53,086 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ed988a340607d50 with lease ID 0x16c1df3a3380a366: from storage DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319 node DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:49:53,086 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3ed988a340607d50 with lease ID 0x16c1df3a3380a366: Processing first storage report for DS-5ba3aa9d-06b7-4599-83ae-8ecfd82235bb from datanode DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503)
2024-12-03T14:49:53,086 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ed988a340607d50 with lease ID 0x16c1df3a3380a366: from storage DS-5ba3aa9d-06b7-4599-83ae-8ecfd82235bb node DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:49:53,125 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,125 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,125 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,125 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 block BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:53,125 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649 block BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:53,125 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta block BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:53,125 WARN  [PacketResponder: BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42953] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,126 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,126 WARN  [PacketResponder: BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42953] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,126 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716 block BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:53,126 WARN  [PacketResponder: BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42953] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_777053275_22 at /127.0.0.1:50428 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:37547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50428 dst: /127.0.0.1:37547
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50412 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50412 dst: /127.0.0.1:37547
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:50372 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50372 dst: /127.0.0.1:37547
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50398 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50398 dst: /127.0.0.1:37547
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:57974 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42953:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57974 dst: /127.0.0.1:42953
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,128 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_777053275_22 at /127.0.0.1:58048 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:42953:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58048 dst: /127.0.0.1:42953
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,128 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:58018 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42953:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58018 dst: /127.0.0.1:42953
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:58004 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42953:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58004 dst: /127.0.0.1:42953
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,130 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75de815b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:53,130 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@fe8f7b0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:53,130 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:53,131 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d3d4ef0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:53,131 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75096fee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:53,132 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:53,132 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:53,133 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-133690028-172.17.0.2-1733237380503 (Datanode Uuid eabbce3a-c616-4ab8-93bd-18ce1ba3c675) service to localhost/127.0.0.1:35825
2024-12-03T14:49:53,133 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:53,133 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data3/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:53,134 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data4/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:53,134 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:53,136 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 block BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,137 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@7512e08c {}] datanode.DataXceiver(331): 127.0.0.1:37547:DataXceiver error processing unknown operation  src: /127.0.0.1:34786 dst: /127.0.0.1:37547
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,137 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:34772 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34772 dst: /127.0.0.1:37547
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,137 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta block BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,141 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,141 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741830_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,141 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_777053275_22 at /127.0.0.1:34778 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:37547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34778 dst: /127.0.0.1:37547
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:53,143 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@24befc55{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:53,144 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@259c861e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:53,144 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:53,144 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35a03ba9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:53,144 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49b91397{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:53,145 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:53,145 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:53,145 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-133690028-172.17.0.2-1733237380503 (Datanode Uuid fe42c28d-56fe-487f-a8fe-815b840a6db0) service to localhost/127.0.0.1:35825
2024-12-03T14:49:53,145 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:53,146 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data1/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:53,146 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data2/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:53,146 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa., hostname=a5d22df9eca2,34883,1733237381195, seqNum=2]
2024-12-03T14:49:53,150 ERROR [FSHLog-0-hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187-prefix:a5d22df9eca2,34883,1733237381195 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,150 WARN  [FSHLog-0-hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187-prefix:a5d22df9eca2,34883,1733237381195 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,151 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,151 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C34883%2C1733237381195:(num 1733237382068) roll requested
2024-12-03T14:49:53,151 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.1733237393151
2024-12-03T14:49:53,158 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:53,158 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:53,158 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:53,158 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:53,158 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:53,158 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237393151
2024-12-03T14:49:53,159 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,159 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:53,159 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35509:35509),(127.0.0.1/127.0.0.1:37891:37891)]
2024-12-03T14:49:53,159 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 is not closed yet, will try archiving it next time
2024-12-03T14:49:53,160 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-03T14:49:53,160 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-03T14:49:53,160 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068
2024-12-03T14:49:53,163 WARN  [IPC Server handler 3 on default port 35825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-12-03T14:49:53,166 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 after 4ms
2024-12-03T14:49:53,687 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:54,508 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:55,160 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:55,161 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237393151
2024-12-03T14:49:55,162 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:55,163 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237393151 block BP-133690028-172.17.0.2-1733237380503:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:49:55,164 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:59340 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:39769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59340 dst: /127.0.0.1:39769
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:55,165 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:55010 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:40629:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55010 dst: /127.0.0.1:40629
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:55,167 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5f24e79d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:55,168 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3986ff43{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:55,168 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:55,168 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ca4b7c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:55,169 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@547f223f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:55,170 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:55,170 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:55,170 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-133690028-172.17.0.2-1733237380503 (Datanode Uuid 1dd964a6-6aaa-4e71-9d31-7046659ed1df) service to localhost/127.0.0.1:35825
2024-12-03T14:49:55,170 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:55,170 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data7/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:55,170 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data8/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:55,171 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:55,687 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:56,508 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:57,160 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:57,162 WARN  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]
2024-12-03T14:49:57,162 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C34883%2C1733237381195:(num 1733237393151) roll requested
2024-12-03T14:49:57,163 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.1733237397162
2024-12-03T14:49:57,168 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 after 4008ms
2024-12-03T14:49:57,170 WARN  [Thread-915 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42953
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:57,170 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:33906 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741839_1021] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741839_1021 to mirror 127.0.0.1:42953
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:57,170 WARN  [Thread-915 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:57,170 WARN  [Thread-915 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741839_1021
2024-12-03T14:49:57,170 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:33906 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741839_1021] {}] datanode.BlockReceiver(316): Block 1073741839 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:49:57,170 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:33906 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33906 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:57,173 WARN  [Thread-915 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:49:57,175 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-03T14:49:57,181 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:57,181 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:57,181 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:57,181 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:57,181 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:57,181 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237393151 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237397162
2024-12-03T14:49:57,183 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37891:37891),(127.0.0.1/127.0.0.1:45789:45789)]
2024-12-03T14:49:57,183 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 is not closed yet, will try archiving it next time
2024-12-03T14:49:57,183 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237393151 is not closed yet, will try archiving it next time
2024-12-03T14:49:57,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40629 is added to blk_1073741838_1020 (size=3600)
2024-12-03T14:49:57,585 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 is not closed yet, will try archiving it next time
2024-12-03T14:49:57,688 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:58,509 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:58,837 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741838_1020 (size=3600)
2024-12-03T14:49:59,180 WARN  [ResponseProcessor for block BP-133690028-172.17.0.2-1733237380503:blk_1073741840_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-133690028-172.17.0.2-1733237380503:blk_1073741840_1022
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,180 WARN  [DataStreamer for file /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237397162 block BP-133690028-172.17.0.2-1733237380503:blk_1073741840_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:49:59,181 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:55034 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:40629:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55034 dst: /127.0.0.1:40629
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,181 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:33908 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33908 dst: /127.0.0.1:36419
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,183 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,183 WARN  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]
2024-12-03T14:49:59,183 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C34883%2C1733237381195:(num 1733237397162) roll requested
2024-12-03T14:49:59,184 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.1733237399183
2024-12-03T14:49:59,185 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2488e9f7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:49:59,189 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@132d95f4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:49:59,189 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:49:59,189 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b53e4f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:49:59,189 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a38f74a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,STOPPED}
2024-12-03T14:49:59,190 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,191 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:49:59,191 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:49:59,191 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741841_1024
2024-12-03T14:49:59,191 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:49:59,191 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:49:59,191 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-133690028-172.17.0.2-1733237380503 (Datanode Uuid 8974d62f-a711-4f42-8566-d2331a5ea5bc) service to localhost/127.0.0.1:35825
2024-12-03T14:49:59,191 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data5/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:59,192 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data6/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:49:59,192 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:49:59,192 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:49:59,194 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42953
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,194 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53352 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741842_1025] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741842_1025 to mirror 127.0.0.1:42953
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,195 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:59,195 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741842_1025
2024-12-03T14:49:59,195 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53352 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741842_1025] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:49:59,195 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53352 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53352 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,195 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:49:59,197 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,197 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:49:59,197 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741843_1026
2024-12-03T14:49:59,198 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:49:59,202 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40629
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,202 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53366 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741844_1027 to mirror 127.0.0.1:40629
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,202 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:49:59,202 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741844_1027
2024-12-03T14:49:59,202 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53366 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:49:59,202 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53366 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53366 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,203 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:49:59,204 WARN  [IPC Server handler 0 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T14:49:59,204 WARN  [IPC Server handler 0 on default port 35825 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T14:49:59,204 WARN  [IPC Server handler 0 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T14:49:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34883 {}] regionserver.HRegion(8855): Flush requested on a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:59,204 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a67ed774ca7dff4fd79fd8580a3231fa 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:49:59,215 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:59,215 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:59,215 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:59,215 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:59,215 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:49:59,215 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237397162 with entries=9, filesize=9.53 KB; new WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237399183
2024-12-03T14:49:59,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741840_1023 (size=9768)
2024-12-03T14:49:59,224 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45789:45789)]
2024-12-03T14:49:59,224 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 is not closed yet, will try archiving it next time
2024-12-03T14:49:59,224 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237397162 is not closed yet, will try archiving it next time
2024-12-03T14:49:59,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/771f597859cb442b8940623091f7fc53 is 1080, key is row0002/info:/1733237395172/Put/seqid=0
2024-12-03T14:49:59,227 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37547
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,227 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53370 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741846_1029 to mirror 127.0.0.1:37547
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,228 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:49:59,228 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741846_1029
2024-12-03T14:49:59,228 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53370 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T14:49:59,228 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53370 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53370 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:49:59,229 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:49:59,230 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,230 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:59,230 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741847_1030
2024-12-03T14:49:59,231 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:49:59,232 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,232 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:49:59,232 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741848_1031
2024-12-03T14:49:59,232 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:49:59,233 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,233 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:49:59,233 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741849_1032
2024-12-03T14:49:59,234 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:49:59,234 WARN  [IPC Server handler 3 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T14:49:59,234 WARN  [IPC Server handler 3 on default port 35825 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T14:49:59,234 WARN  [IPC Server handler 3 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T14:49:59,237 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741850_1033 (size=10347)
2024-12-03T14:49:59,619 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 is not closed yet, will try archiving it next time
2024-12-03T14:49:59,638 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/771f597859cb442b8940623091f7fc53
2024-12-03T14:49:59,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/771f597859cb442b8940623091f7fc53 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/771f597859cb442b8940623091f7fc53
2024-12-03T14:49:59,651 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/771f597859cb442b8940623091f7fc53, entries=5, sequenceid=11, filesize=10.1 K
2024-12-03T14:49:59,652 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for a67ed774ca7dff4fd79fd8580a3231fa in 448ms, sequenceid=11, compaction requested=false
2024-12-03T14:49:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:49:59,688 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34883 {}] regionserver.HRegion(8855): Flush requested on a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:49:59,841 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a67ed774ca7dff4fd79fd8580a3231fa 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-03T14:49:59,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/6d3afceb404148ea8cb368ff902e94e3 is 1080, key is row0007/info:/1733237399206/Put/seqid=0
2024-12-03T14:49:59,852 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,852 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:49:59,852 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741851_1034
2024-12-03T14:49:59,853 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:49:59,854 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,854 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:49:59,854 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741852_1035
2024-12-03T14:49:59,855 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:49:59,856 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,857 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:49:59,857 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741853_1036
2024-12-03T14:49:59,857 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:49:59,858 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:49:59,858 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:49:59,858 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741854_1037
2024-12-03T14:49:59,859 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:49:59,860 WARN  [IPC Server handler 4 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T14:49:59,860 WARN  [IPC Server handler 4 on default port 35825 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T14:49:59,860 WARN  [IPC Server handler 4 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T14:49:59,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741855_1038 (size=12506)
2024-12-03T14:50:00,264 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/6d3afceb404148ea8cb368ff902e94e3
2024-12-03T14:50:00,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/6d3afceb404148ea8cb368ff902e94e3 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3
2024-12-03T14:50:00,281 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3, entries=7, sequenceid=24, filesize=12.2 K
2024-12-03T14:50:00,282 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for a67ed774ca7dff4fd79fd8580a3231fa in 441ms, sequenceid=24, compaction requested=false
2024-12-03T14:50:00,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:50:00,282 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-03T14:50:00,282 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:00,282 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3 because midkey is the same as first or last row
2024-12-03T14:50:00,509 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,224 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,225 WARN  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]
2024-12-03T14:50:01,225 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C34883%2C1733237381195:(num 1733237399183) roll requested
2024-12-03T14:50:01,226 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.1733237401225
2024-12-03T14:50:01,232 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,232 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:01,232 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741856_1039
2024-12-03T14:50:01,233 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:01,236 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42953
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,236 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53392 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741857_1040 to mirror 127.0.0.1:42953
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,236 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:50:01,236 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741857_1040
2024-12-03T14:50:01,236 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53392 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:50:01,237 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53392 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53392 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,237 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:50:01,240 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37547
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,240 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53402 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741858_1041 to mirror 127.0.0.1:37547
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,240 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:01,240 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741858_1041
2024-12-03T14:50:01,240 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53402 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:50:01,240 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53402 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53402 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,241 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:01,242 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,242 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:01,242 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741859_1042
2024-12-03T14:50:01,243 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:01,244 WARN  [IPC Server handler 2 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T14:50:01,244 WARN  [IPC Server handler 2 on default port 35825 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T14:50:01,244 WARN  [IPC Server handler 2 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T14:50:01,247 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:01,247 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:01,247 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:01,248 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:01,248 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:01,248 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237399183 with entries=15, filesize=14.79 KB; new WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237401225
2024-12-03T14:50:01,249 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45789:45789)]
2024-12-03T14:50:01,249 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 is not closed yet, will try archiving it next time
2024-12-03T14:50:01,249 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237399183 is not closed yet, will try archiving it next time
2024-12-03T14:50:01,250 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237393151 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs/a5d22df9eca2%2C34883%2C1733237381195.1733237393151
2024-12-03T14:50:01,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741845_1028 (size=15148)
2024-12-03T14:50:01,252 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237397162 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs/a5d22df9eca2%2C34883%2C1733237381195.1733237397162
2024-12-03T14:50:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34883 {}] regionserver.HRegion(8855): Flush requested on a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:50:01,281 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a67ed774ca7dff4fd79fd8580a3231fa 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-03T14:50:01,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/6ca6159b7b94477a877caff270caf165 is 1079, key is tmprow/info:/1733237401280/Put/seqid=0
2024-12-03T14:50:01,288 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,288 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:01,288 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741861_1044
2024-12-03T14:50:01,288 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:01,290 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,290 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:50:01,290 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741862_1045
2024-12-03T14:50:01,290 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:50:01,291 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,291 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:01,291 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741863_1046
2024-12-03T14:50:01,292 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:01,293 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,293 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:01,293 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741864_1047
2024-12-03T14:50:01,293 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:01,294 WARN  [IPC Server handler 1 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T14:50:01,294 WARN  [IPC Server handler 1 on default port 35825 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T14:50:01,294 WARN  [IPC Server handler 1 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T14:50:01,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741865_1048 (size=6027)
2024-12-03T14:50:01,653 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 is not closed yet, will try archiving it next time
2024-12-03T14:50:01,688 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,698 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/6ca6159b7b94477a877caff270caf165
2024-12-03T14:50:01,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/6ca6159b7b94477a877caff270caf165 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6ca6159b7b94477a877caff270caf165
2024-12-03T14:50:01,716 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6ca6159b7b94477a877caff270caf165, entries=1, sequenceid=34, filesize=5.9 K
2024-12-03T14:50:01,717 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for a67ed774ca7dff4fd79fd8580a3231fa in 436ms, sequenceid=34, compaction requested=true
2024-12-03T14:50:01,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:50:01,717 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-03T14:50:01,717 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:01,717 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3 because midkey is the same as first or last row
2024-12-03T14:50:01,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a67ed774ca7dff4fd79fd8580a3231fa:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:50:01,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:50:01,718 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:50:01,719 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:50:01,719 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HStore(1541): a67ed774ca7dff4fd79fd8580a3231fa/info is initiating minor compaction (all files)
2024-12-03T14:50:01,719 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a67ed774ca7dff4fd79fd8580a3231fa/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:50:01,719 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/771f597859cb442b8940623091f7fc53, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6ca6159b7b94477a877caff270caf165] into tmpdir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp, totalSize=28.2 K
2024-12-03T14:50:01,720 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771f597859cb442b8940623091f7fc53, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733237395172
2024-12-03T14:50:01,720 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3afceb404148ea8cb368ff902e94e3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733237399206
2024-12-03T14:50:01,721 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ca6159b7b94477a877caff270caf165, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733237401280
2024-12-03T14:50:01,734 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a67ed774ca7dff4fd79fd8580a3231fa#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:50:01,735 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/f7fff76e9b544d959b1335d024fd8f14 is 1080, key is row0002/info:/1733237395172/Put/seqid=0
2024-12-03T14:50:01,737 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,737 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:50:01,737 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741866_1049
2024-12-03T14:50:01,737 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:50:01,738 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,739 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:01,739 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741867_1050
2024-12-03T14:50:01,739 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:01,741 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40629
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,741 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53442 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741868_1051 to mirror 127.0.0.1:40629
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,741 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:01,741 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53442 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T14:50:01,741 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741868_1051
2024-12-03T14:50:01,741 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53442 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53442 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,742 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:01,744 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39769
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:01,744 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53458 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741869_1052 to mirror 127.0.0.1:39769
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,744 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:01,745 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741869_1052
2024-12-03T14:50:01,745 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53458 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T14:50:01,745 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:53458 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53458 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:01,745 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:01,746 WARN  [IPC Server handler 0 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T14:50:01,746 WARN  [IPC Server handler 0 on default port 35825 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T14:50:01,746 WARN  [IPC Server handler 0 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T14:50:01,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741870_1053 (size=17994)
2024-12-03T14:50:02,090 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@a90acb9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741840_1023 to 127.0.0.1:39769 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:02,090 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@38b1dd8d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741850_1033 to 127.0.0.1:37547 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:02,157 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/f7fff76e9b544d959b1335d024fd8f14 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14
2024-12-03T14:50:02,163 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a67ed774ca7dff4fd79fd8580a3231fa/info of a67ed774ca7dff4fd79fd8580a3231fa into f7fff76e9b544d959b1335d024fd8f14(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:50:02,164 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa., storeName=a67ed774ca7dff4fd79fd8580a3231fa/info, priority=13, startTime=1733237401717; duration=0sec
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14 because midkey is the same as first or last row
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14 because midkey is the same as first or last row
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14 because midkey is the same as first or last row
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:50:02,164 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a67ed774ca7dff4fd79fd8580a3231fa:info
2024-12-03T14:50:02,510 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34883 {}] regionserver.HRegion(8855): Flush requested on a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:50:02,705 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a67ed774ca7dff4fd79fd8580a3231fa 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-03T14:50:02,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/53a2a2059f5c4ebeba5efc0a064bfed3 is 1079, key is tmprow/info:/1733237402704/Put/seqid=0
2024-12-03T14:50:02,712 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:02,712 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:02,712 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741871_1054
2024-12-03T14:50:02,713 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:02,714 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:02,714 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:02,714 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741872_1055
2024-12-03T14:50:02,715 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:02,716 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:02,716 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]) is bad.
2024-12-03T14:50:02,716 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741873_1056
2024-12-03T14:50:02,717 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42953,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]
2024-12-03T14:50:02,718 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:02,718 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:02,718 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741874_1057
2024-12-03T14:50:02,719 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:02,720 WARN  [IPC Server handler 4 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T14:50:02,720 WARN  [IPC Server handler 4 on default port 35825 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T14:50:02,720 WARN  [IPC Server handler 4 on default port 35825 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T14:50:02,723 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741875_1058 (size=6027)
2024-12-03T14:50:03,087 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@38b1dd8d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741855_1038 to 127.0.0.1:42953 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:03,087 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@a90acb9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741845_1028 to 127.0.0.1:42953 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:03,124 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/53a2a2059f5c4ebeba5efc0a064bfed3
2024-12-03T14:50:03,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/53a2a2059f5c4ebeba5efc0a064bfed3 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/53a2a2059f5c4ebeba5efc0a064bfed3
2024-12-03T14:50:03,138 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/53a2a2059f5c4ebeba5efc0a064bfed3, entries=1, sequenceid=45, filesize=5.9 K
2024-12-03T14:50:03,139 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for a67ed774ca7dff4fd79fd8580a3231fa in 434ms, sequenceid=45, compaction requested=false
2024-12-03T14:50:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:50:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-03T14:50:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14 because midkey is the same as first or last row
2024-12-03T14:50:03,250 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:03,250 WARN  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-03T14:50:03,322 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:03,325 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:03,325 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:03,325 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:03,326 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:50:03,326 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7bfce1ef{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:03,326 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@798caa79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:03,416 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@32cdb76b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/java.io.tmpdir/jetty-localhost-44511-hadoop-hdfs-3_4_1-tests_jar-_-any-9499014495622977355/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:03,416 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2c92a539{HTTP/1.1, (http/1.1)}{localhost:44511}
2024-12-03T14:50:03,416 INFO  [Time-limited test {}] server.Server(415): Started @126723ms
2024-12-03T14:50:03,418 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:50:03,490 WARN  [Thread-983 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:50:03,498 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbee9dea83cc66448 with lease ID 0x16c1df3a3380a367: from storage DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c node DatanodeRegistration(127.0.0.1:41117, datanodeUuid=eabbce3a-c616-4ab8-93bd-18ce1ba3c675, infoPort=36511, infoSecurePort=0, ipcPort=35019, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:50:03,498 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbee9dea83cc66448 with lease ID 0x16c1df3a3380a367: from storage DS-8466e49a-b879-42ce-abdc-a1a6471ed4d1 node DatanodeRegistration(127.0.0.1:41117, datanodeUuid=eabbce3a-c616-4ab8-93bd-18ce1ba3c675, infoPort=36511, infoSecurePort=0, ipcPort=35019, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:03,689 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:04,510 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:05,091 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@38b1dd8d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741870_1053 to 127.0.0.1:39769 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:05,095 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741865_1048 (size=6027)
2024-12-03T14:50:05,251 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:05,689 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:06,091 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@a90acb9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741875_1058 to 127.0.0.1:40629 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:06,511 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:07,251 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:07,690 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:08,511 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:09,252 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:09,691 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:10,512 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,135 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T14:50:11,253 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,691 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,821 ERROR [FSHLog-0-hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData-prefix:a5d22df9eca2,37049,1733237381157 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,821 WARN  [FSHLog-0-hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData-prefix:a5d22df9eca2,37049,1733237381157 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,822 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C37049%2C1733237381157:(num 1733237381716) roll requested
2024-12-03T14:50:11,822 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C37049%2C1733237381157.1733237411822
2024-12-03T14:50:11,830 WARN  [Thread-1004 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40629
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:59432 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data4]'}, localName='127.0.0.1:41117', datanodeUuid='eabbce3a-c616-4ab8-93bd-18ce1ba3c675', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741876_1059 to mirror 127.0.0.1:40629
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:11,830 WARN  [Thread-1004 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:11,830 WARN  [Thread-1004 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741876_1059
2024-12-03T14:50:11,830 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:59432 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:50:11,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:59432 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:41117:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59432 dst: /127.0.0.1:41117
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:11,831 WARN  [Thread-1004 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:11,835 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:45076 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741877_1060 to mirror 127.0.0.1:39769
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:11,835 WARN  [Thread-1004 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39769
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,835 WARN  [Thread-1004 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:11,835 WARN  [Thread-1004 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741877_1060
2024-12-03T14:50:11,835 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:45076 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:50:11,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1369592741_22 at /127.0.0.1:45076 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45076 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:11,836 WARN  [Thread-1004 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:11,842 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:11,842 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:11,842 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:11,842 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:11,843 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:11,843 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237411822
2024-12-03T14:50:11,843 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,844 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:11,844 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
2024-12-03T14:50:11,844 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45789:45789),(127.0.0.1/127.0.0.1:36511:36511)]
2024-12-03T14:50:11,844 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716 is not closed yet, will try archiving it next time
2024-12-03T14:50:11,844 WARN  [IPC Server handler 1 on default port 35825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716 has not been closed. Lease recovery is in progress. RecoveryId = 1062 for block blk_1073741830_1014
2024-12-03T14:50:11,845 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716 after 1ms
2024-12-03T14:50:12,512 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:13,253 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:13,514 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@52c27844 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:37547,null,null])
java.net.ConnectException: Call From a5d22df9eca2/172.17.0.2 to localhost:46555 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-03T14:50:13,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741833_1019 (size=455)
2024-12-03T14:50:14,195 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237382068 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs/a5d22df9eca2%2C34883%2C1733237381195.1733237382068
2024-12-03T14:50:14,196 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237399183 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs/a5d22df9eca2%2C34883%2C1733237381195.1733237399183
2024-12-03T14:50:14,497 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b5ee87b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41117, datanodeUuid=eabbce3a-c616-4ab8-93bd-18ce1ba3c675, infoPort=36511, infoSecurePort=0, ipcPort=35019, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741833_1019 to 127.0.0.1:39769 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:14,513 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:15,254 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:15,848 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716 after 4003ms
2024-12-03T14:50:16,513 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:17,255 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,514 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,865 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.1733237418865
2024-12-03T14:50:18,870 WARN  [Thread-1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,871 WARN  [Thread-1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741879_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:18,871 WARN  [Thread-1015 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741879_1063
2024-12-03T14:50:18,871 WARN  [Thread-1015 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:18,873 WARN  [Thread-1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,873 WARN  [Thread-1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741880_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:18,873 WARN  [Thread-1015 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741880_1064
2024-12-03T14:50:18,874 WARN  [Thread-1015 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:18,879 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:18,879 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:18,880 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:18,880 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:18,880 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:18,880 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237401225 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237418865
2024-12-03T14:50:18,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45789:45789),(127.0.0.1/127.0.0.1:36511:36511)]
2024-12-03T14:50:18,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237401225 is not closed yet, will try archiving it next time
2024-12-03T14:50:18,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741860_1043 (size=13591)
2024-12-03T14:50:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34883 {}] regionserver.HRegion(8855): Flush requested on a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:50:18,895 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a67ed774ca7dff4fd79fd8580a3231fa 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-03T14:50:18,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/e20dfc934fe54b53a3921ebaef8e6f13 is 1080, key is row0013/info:/1733237418883/Put/seqid=0
2024-12-03T14:50:18,901 WARN  [Thread-1021 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,901 WARN  [Thread-1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741882_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:18,901 WARN  [Thread-1021 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741882_1066
2024-12-03T14:50:18,902 WARN  [Thread-1021 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:18,904 WARN  [Thread-1021 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,904 WARN  [Thread-1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741883_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:18,904 WARN  [Thread-1021 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741883_1067
2024-12-03T14:50:18,904 WARN  [Thread-1021 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:18,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741884_1068 (size=11421)
2024-12-03T14:50:18,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741884_1068 (size=11421)
2024-12-03T14:50:18,913 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/e20dfc934fe54b53a3921ebaef8e6f13
2024-12-03T14:50:18,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/e20dfc934fe54b53a3921ebaef8e6f13 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/e20dfc934fe54b53a3921ebaef8e6f13
2024-12-03T14:50:18,926 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/e20dfc934fe54b53a3921ebaef8e6f13, entries=6, sequenceid=55, filesize=11.2 K
2024-12-03T14:50:18,927 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7530, heapSize ~8.11 KB/8304, currentSize=6.30 KB/6455 for a67ed774ca7dff4fd79fd8580a3231fa in 32ms, sequenceid=55, compaction requested=true
2024-12-03T14:50:18,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:50:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.6 K, sizeToCheck=16.0 K
2024-12-03T14:50:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14 because midkey is the same as first or last row
2024-12-03T14:50:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a67ed774ca7dff4fd79fd8580a3231fa:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:50:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:50:18,928 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:50:18,929 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 35442 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:50:18,930 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HStore(1541): a67ed774ca7dff4fd79fd8580a3231fa/info is initiating minor compaction (all files)
2024-12-03T14:50:18,930 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a67ed774ca7dff4fd79fd8580a3231fa/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:50:18,930 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/53a2a2059f5c4ebeba5efc0a064bfed3, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/e20dfc934fe54b53a3921ebaef8e6f13] into tmpdir=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp, totalSize=34.6 K
2024-12-03T14:50:18,930 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7fff76e9b544d959b1335d024fd8f14, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733237395172
2024-12-03T14:50:18,931 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a2a2059f5c4ebeba5efc0a064bfed3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733237402704
2024-12-03T14:50:18,931 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20dfc934fe54b53a3921ebaef8e6f13, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733237403110
2024-12-03T14:50:18,949 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a67ed774ca7dff4fd79fd8580a3231fa#info#compaction#24 average throughput is 8.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:50:18,949 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/4bf3a009414b4f7ebab03a9fbe634817 is 1080, key is row0002/info:/1733237395172/Put/seqid=0
2024-12-03T14:50:18,951 WARN  [Thread-1030 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,951 WARN  [Thread-1030 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741885_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:18,951 WARN  [Thread-1030 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741885_1069
2024-12-03T14:50:18,952 WARN  [Thread-1030 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:18,953 WARN  [Thread-1030 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:18,953 WARN  [Thread-1030 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741886_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:18,953 WARN  [Thread-1030 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741886_1070
2024-12-03T14:50:18,954 WARN  [Thread-1030 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:18,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741887_1071 (size=23502)
2024-12-03T14:50:18,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741887_1071 (size=23502)
2024-12-03T14:50:18,966 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/4bf3a009414b4f7ebab03a9fbe634817 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/4bf3a009414b4f7ebab03a9fbe634817
2024-12-03T14:50:18,976 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a67ed774ca7dff4fd79fd8580a3231fa/info of a67ed774ca7dff4fd79fd8580a3231fa into 4bf3a009414b4f7ebab03a9fbe634817(size=23.0 K), total size for store is 23.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:50:18,976 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:50:18,976 INFO  [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa., storeName=a67ed774ca7dff4fd79fd8580a3231fa/info, priority=13, startTime=1733237418928; duration=0sec
2024-12-03T14:50:18,976 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K
2024-12-03T14:50:18,976 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:18,976 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/4bf3a009414b4f7ebab03a9fbe634817 because midkey is the same as first or last row
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/4bf3a009414b4f7ebab03a9fbe634817 because midkey is the same as first or last row
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/4bf3a009414b4f7ebab03a9fbe634817 because midkey is the same as first or last row
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:50:18,977 DEBUG [RS:0;a5d22df9eca2:34883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a67ed774ca7dff4fd79fd8580a3231fa:info
2024-12-03T14:50:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34883 {}] regionserver.HRegion(8855): Flush requested on a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:50:19,116 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a67ed774ca7dff4fd79fd8580a3231fa 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-03T14:50:19,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/df00eede11464a7ea9545e2491d5d2c1 is 1080, key is row0018/info:/1733237418896/Put/seqid=0
2024-12-03T14:50:19,126 WARN  [Thread-1036 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1072
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,126 WARN  [Thread-1036 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741888_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:19,126 WARN  [Thread-1036 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741888_1072
2024-12-03T14:50:19,127 WARN  [Thread-1036 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:19,129 WARN  [Thread-1036 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40629
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,129 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:42856 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741889_1073] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10]'}, localName='127.0.0.1:36419', datanodeUuid='78320ed1-34e3-4adc-99ae-95eb16df670a', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741889_1073 to mirror 127.0.0.1:40629
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:19,129 WARN  [Thread-1036 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:19,129 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:42856 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741889_1073] {}] datanode.BlockReceiver(316): Block 1073741889 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T14:50:19,130 WARN  [Thread-1036 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741889_1073
2024-12-03T14:50:19,130 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:42856 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741889_1073] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42856 dst: /127.0.0.1:36419
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:19,130 WARN  [Thread-1036 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:19,131 WARN  [Thread-1036 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,132 WARN  [Thread-1036 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:19,132 WARN  [Thread-1036 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741890_1074
2024-12-03T14:50:19,132 WARN  [Thread-1036 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:19,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741891_1075 (size=11421)
2024-12-03T14:50:19,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741891_1075 (size=11421)
2024-12-03T14:50:19,138 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/df00eede11464a7ea9545e2491d5d2c1
2024-12-03T14:50:19,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/.tmp/info/df00eede11464a7ea9545e2491d5d2c1 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/df00eede11464a7ea9545e2491d5d2c1
2024-12-03T14:50:19,152 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/df00eede11464a7ea9545e2491d5d2c1, entries=6, sequenceid=66, filesize=11.2 K
2024-12-03T14:50:19,153 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7530, heapSize ~8.11 KB/8304, currentSize=0 B/0 for a67ed774ca7dff4fd79fd8580a3231fa in 37ms, sequenceid=66, compaction requested=false
2024-12-03T14:50:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a67ed774ca7dff4fd79fd8580a3231fa:

2024-12-03T14:50:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.1 K, sizeToCheck=16.0 K
2024-12-03T14:50:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:50:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/4bf3a009414b4f7ebab03a9fbe634817 because midkey is the same as first or last row
2024-12-03T14:50:19,256 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,256 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-03T14:50:19,284 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.1733237401225 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs/a5d22df9eca2%2C34883%2C1733237381195.1733237401225
2024-12-03T14:50:19,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-03T14:50:19,317 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:50:19,318 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:50:19,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:19,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:19,318 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-03T14:50:19,319 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T14:50:19,319 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1048628396, stopped=false
2024-12-03T14:50:19,319 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,37049,1733237381157
2024-12-03T14:50:19,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:50:19,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:50:19,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:50:19,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:19,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:19,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:19,322 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:50:19,323 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:50:19,323 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:50:19,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:19,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:50:19,324 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,34883,1733237381195' *****
2024-12-03T14:50:19,324 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:50:19,324 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:50:19,324 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,44475,1733237382458' *****
2024-12-03T14:50:19,324 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:50:19,324 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:50:19,324 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:50:19,324 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:50:19,324 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:50:19,324 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:50:19,324 INFO  [RS:1;a5d22df9eca2:44475 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:50:19,324 INFO  [RS:0;a5d22df9eca2:34883 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:50:19,324 INFO  [RS:1;a5d22df9eca2:44475 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:50:19,324 INFO  [RS:0;a5d22df9eca2:34883 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:50:19,324 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,44475,1733237382458
2024-12-03T14:50:19,324 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:50:19,324 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(3091): Received CLOSE for a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:50:19,325 INFO  [RS:1;a5d22df9eca2:44475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a5d22df9eca2:44475.
2024-12-03T14:50:19,325 DEBUG [RS:1;a5d22df9eca2:44475 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:50:19,325 DEBUG [RS:1;a5d22df9eca2:44475 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:19,325 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,34883,1733237381195
2024-12-03T14:50:19,325 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:50:19,325 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,44475,1733237382458; all regions closed.
2024-12-03T14:50:19,325 INFO  [RS:0;a5d22df9eca2:34883 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:34883.
2024-12-03T14:50:19,325 DEBUG [RS:0;a5d22df9eca2:34883 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:50:19,325 DEBUG [RS:0;a5d22df9eca2:34883 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:19,325 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a67ed774ca7dff4fd79fd8580a3231fa, disabling compactions & flushes
2024-12-03T14:50:19,325 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:50:19,325 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:50:19,325 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:50:19,325 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:50:19,325 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:50:19,325 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa. after waiting 0 ms
2024-12-03T14:50:19,326 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-03T14:50:19,326 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:50:19,326 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,326 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,326 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,326 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,326 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-03T14:50:19,326 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1325): Online Regions={a67ed774ca7dff4fd79fd8580a3231fa=TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa., 1588230740=hbase:meta,,1.1588230740}
2024-12-03T14:50:19,326 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,326 DEBUG [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a67ed774ca7dff4fd79fd8580a3231fa
2024-12-03T14:50:19,326 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/771f597859cb442b8940623091f7fc53, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6ca6159b7b94477a877caff270caf165, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/53a2a2059f5c4ebeba5efc0a064bfed3, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/e20dfc934fe54b53a3921ebaef8e6f13] to archive
2024-12-03T14:50:19,326 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:50:19,326 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:50:19,326 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:50:19,327 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:50:19,327 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:50:19,327 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,327 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,327 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
2024-12-03T14:50:19,327 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-03T14:50:19,327 WARN  [IPC Server handler 2 on default port 35825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649 has not been closed. Lease recovery is in progress. RecoveryId = 1076 for block blk_1073741837_1015
2024-12-03T14:50:19,327 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T14:50:19,327 ERROR [FSHLog-0-hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187-prefix:a5d22df9eca2,34883,1733237381195.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,328 WARN  [FSHLog-0-hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187-prefix:a5d22df9eca2,34883,1733237381195.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,328 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649 after 1ms
2024-12-03T14:50:19,328 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C34883%2C1733237381195.meta:.meta(num 1733237382289) roll requested
2024-12-03T14:50:19,328 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34883%2C1733237381195.meta.1733237419328.meta
2024-12-03T14:50:19,330 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/771f597859cb442b8940623091f7fc53 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/771f597859cb442b8940623091f7fc53
2024-12-03T14:50:19,331 WARN  [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1077
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,332 WARN  [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741892_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:19,332 WARN  [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741892_1077
2024-12-03T14:50:19,332 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6d3afceb404148ea8cb368ff902e94e3
2024-12-03T14:50:19,332 WARN  [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:19,333 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/f7fff76e9b544d959b1335d024fd8f14
2024-12-03T14:50:19,333 WARN  [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,333 WARN  [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741893_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:19,333 WARN  [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741893_1078
2024-12-03T14:50:19,334 WARN  [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:19,335 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6ca6159b7b94477a877caff270caf165 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/6ca6159b7b94477a877caff270caf165
2024-12-03T14:50:19,336 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/53a2a2059f5c4ebeba5efc0a064bfed3 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/53a2a2059f5c4ebeba5efc0a064bfed3
2024-12-03T14:50:19,336 WARN  [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1079
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40629
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,336 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50058 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741894_1079] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data4]'}, localName='127.0.0.1:41117', datanodeUuid='eabbce3a-c616-4ab8-93bd-18ce1ba3c675', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741894_1079 to mirror 127.0.0.1:40629
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:19,336 WARN  [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741894_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:19,336 WARN  [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741894_1079
2024-12-03T14:50:19,336 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50058 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741894_1079] {}] datanode.BlockReceiver(316): Block 1073741894 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T14:50:19,337 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50058 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741894_1079] {}] datanode.DataXceiver(331): 127.0.0.1:41117:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50058 dst: /127.0.0.1:41117
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:19,337 WARN  [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:19,337 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/e20dfc934fe54b53a3921ebaef8e6f13 to hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/info/e20dfc934fe54b53a3921ebaef8e6f13
2024-12-03T14:50:19,338 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a5d22df9eca2:37049 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-03T14:50:19,338 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [771f597859cb442b8940623091f7fc53=10347, 6d3afceb404148ea8cb368ff902e94e3=12506, f7fff76e9b544d959b1335d024fd8f14=17994, 6ca6159b7b94477a877caff270caf165=6027, 53a2a2059f5c4ebeba5efc0a064bfed3=6027, e20dfc934fe54b53a3921ebaef8e6f13=11421]
2024-12-03T14:50:19,342 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,342 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,342 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,342 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/default/TestLogRolling-testLogRollOnDatanodeDeath/a67ed774ca7dff4fd79fd8580a3231fa/recovered.edits/69.seqid, newMaxSeqId=69, maxSeqId=1
2024-12-03T14:50:19,342 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,342 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,342 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237419328.meta
2024-12-03T14:50:19,343 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,343 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:50:19,343 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,343 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
2024-12-03T14:50:19,343 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a67ed774ca7dff4fd79fd8580a3231fa:
Waiting for close lock at 1733237419325Running coprocessor pre-close hooks at 1733237419325Disabling compacts and flushes for region at 1733237419325Disabling writes for close at 1733237419326 (+1 ms)Writing region close event to WAL at 1733237419338 (+12 ms)Running coprocessor post-close hooks at 1733237419343 (+5 ms)Closed at 1733237419343
2024-12-03T14:50:19,343 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa.
2024-12-03T14:50:19,343 WARN  [IPC Server handler 4 on default port 35825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta has not been closed. Lease recovery is in progress. RecoveryId = 1081 for block blk_1073741834_1010
2024-12-03T14:50:19,343 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta after 0ms
2024-12-03T14:50:19,348 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45789:45789),(127.0.0.1/127.0.0.1:36511:36511)]
2024-12-03T14:50:19,348 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta is not closed yet, will try archiving it next time
2024-12-03T14:50:19,364 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/info/0e377b85655c4a6685105409da8d3bbc is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733237382547.a67ed774ca7dff4fd79fd8580a3231fa./info:regioninfo/1733237382910/Put/seqid=0
2024-12-03T14:50:19,367 WARN  [Thread-1051 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1082
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37547
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,367 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50076 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741896_1082] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data4]'}, localName='127.0.0.1:41117', datanodeUuid='eabbce3a-c616-4ab8-93bd-18ce1ba3c675', xmitsInProgress=0}:Exception transferring block BP-133690028-172.17.0.2-1733237380503:blk_1073741896_1082 to mirror 127.0.0.1:37547
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:19,367 WARN  [Thread-1051 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741896_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK], DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:19,367 WARN  [Thread-1051 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741896_1082
2024-12-03T14:50:19,367 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50076 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741896_1082] {}] datanode.BlockReceiver(316): Block 1073741896 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T14:50:19,367 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_326129126_22 at /127.0.0.1:50076 [Receiving block BP-133690028-172.17.0.2-1733237380503:blk_1073741896_1082] {}] datanode.DataXceiver(331): 127.0.0.1:41117:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50076 dst: /127.0.0.1:41117
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:19,368 WARN  [Thread-1051 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:19,369 WARN  [Thread-1051 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,369 WARN  [Thread-1051 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741897_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:19,369 WARN  [Thread-1051 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741897_1083
2024-12-03T14:50:19,370 WARN  [Thread-1051 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:19,371 WARN  [Thread-1051 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1084
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,371 WARN  [Thread-1051 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741898_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK], DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]) is bad.
2024-12-03T14:50:19,371 WARN  [Thread-1051 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741898_1084
2024-12-03T14:50:19,371 WARN  [Thread-1051 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39769,DS-86c1a948-a62a-447b-8f80-428cc55823dc,DISK]
2024-12-03T14:50:19,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741899_1085 (size=7089)
2024-12-03T14:50:19,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741899_1085 (size=7089)
2024-12-03T14:50:19,378 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/info/0e377b85655c4a6685105409da8d3bbc
2024-12-03T14:50:19,396 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/ns/eef8d05eb1e143e998f6061019e0c889 is 43, key is default/ns:d/1733237382340/Put/seqid=0
2024-12-03T14:50:19,398 WARN  [Thread-1058 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,398 WARN  [Thread-1058 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:41117,DS-ff2361a9-dd33-4f10-9db5-c4c93ac5f59c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:19,398 WARN  [Thread-1058 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741900_1086
2024-12-03T14:50:19,399 WARN  [Thread-1058 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:19,400 WARN  [Thread-1058 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1087
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,401 WARN  [Thread-1058 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741901_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]) is bad.
2024-12-03T14:50:19,401 WARN  [Thread-1058 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741901_1087
2024-12-03T14:50:19,401 WARN  [Thread-1058 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40629,DS-be8583c9-8a47-4ad5-b311-99bd4c422f49,DISK]
2024-12-03T14:50:19,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741902_1088 (size=5153)
2024-12-03T14:50:19,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741902_1088 (size=5153)
2024-12-03T14:50:19,411 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/ns/eef8d05eb1e143e998f6061019e0c889
2024-12-03T14:50:19,430 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/table/5256dc0792e249cca8fc682df94cc32d is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733237382923/Put/seqid=0
2024-12-03T14:50:19,432 WARN  [Thread-1064 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1089
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:19,432 WARN  [Thread-1064 {}] hdfs.DataStreamer(1731): Error Recovery for BP-133690028-172.17.0.2-1733237380503:blk_1073741903_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-05b1a14a-f7cb-4ae0-a161-104d5fc97319,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]) is bad.
2024-12-03T14:50:19,432 WARN  [Thread-1064 {}] hdfs.DataStreamer(1850): Abandoning BP-133690028-172.17.0.2-1733237380503:blk_1073741903_1089
2024-12-03T14:50:19,433 WARN  [Thread-1064 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37547,DS-e509cf23-c0c7-4e2a-93d4-a7a5f5d80b20,DISK]
2024-12-03T14:50:19,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741904_1090 (size=5424)
2024-12-03T14:50:19,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741904_1090 (size=5424)
2024-12-03T14:50:19,438 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/table/5256dc0792e249cca8fc682df94cc32d
2024-12-03T14:50:19,443 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/info/0e377b85655c4a6685105409da8d3bbc as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/info/0e377b85655c4a6685105409da8d3bbc
2024-12-03T14:50:19,450 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/info/0e377b85655c4a6685105409da8d3bbc, entries=10, sequenceid=11, filesize=6.9 K
2024-12-03T14:50:19,451 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/ns/eef8d05eb1e143e998f6061019e0c889 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/ns/eef8d05eb1e143e998f6061019e0c889
2024-12-03T14:50:19,457 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/ns/eef8d05eb1e143e998f6061019e0c889, entries=2, sequenceid=11, filesize=5.0 K
2024-12-03T14:50:19,458 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/.tmp/table/5256dc0792e249cca8fc682df94cc32d as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/table/5256dc0792e249cca8fc682df94cc32d
2024-12-03T14:50:19,464 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/table/5256dc0792e249cca8fc682df94cc32d, entries=2, sequenceid=11, filesize=5.3 K
2024-12-03T14:50:19,465 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 138ms, sequenceid=11, compaction requested=false
2024-12-03T14:50:19,470 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-03T14:50:19,471 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:50:19,471 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:50:19,471 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237419326Running coprocessor pre-close hooks at 1733237419326Disabling compacts and flushes for region at 1733237419326Disabling writes for close at 1733237419327 (+1 ms)Obtaining lock to block concurrent updates at 1733237419327Preparing flush snapshotting stores in 1588230740 at 1733237419327Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733237419328 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733237419349 (+21 ms)Flushing 1588230740/info: creating writer at 1733237419349Flushing 1588230740/info: appending metadata at 1733237419363 (+14 ms)Flushing 1588230740/info: closing flushed file at 1733237419364 (+1 ms)Flushing 1588230740/ns: creating writer at 1733237419383 (+19 ms)Flushing 1588230740/ns: appending metadata at 1733237419396 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733237419396Flushing 1588230740/table: creating writer at 1733237419417 (+21 ms)Flushing 1588230740/table: appending metadata at 1733237419430 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733237419430Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24996583: reopening flushed file at 1733237419443 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f4d06a1: reopening flushed file at 1733237419450 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c2b6768: reopening flushed file at 1733237419457 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 138ms, sequenceid=11, compaction requested=false at 1733237419465 (+8 ms)Writing region close event to WAL at 1733237419466 (+1 ms)Running coprocessor post-close hooks at 1733237419471 (+5 ms)Closed at 1733237419471
2024-12-03T14:50:19,471 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T14:50:19,526 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,34883,1733237381195; all regions closed.
2024-12-03T14:50:19,527 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,527 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,527 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,527 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,527 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:19,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741895_1080 (size=825)
2024-12-03T14:50:19,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741895_1080 (size=825)
2024-12-03T14:50:19,533 INFO  [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-03T14:50:19,533 INFO  [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-03T14:50:19,914 INFO  [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-03T14:50:19,914 INFO  [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-03T14:50:19,922 INFO  [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:50:20,094 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@a90acb9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=78320ed1-34e3-4adc-99ae-95eb16df670a, infoPort=45789, infoSecurePort=0, ipcPort=36507, storageInfo=lv=-57;cid=testClusterID;nsid=31966541;c=1733237380503):Failed to transfer BP-133690028-172.17.0.2-1733237380503:blk_1073741860_1043 to 127.0.0.1:37547 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:20,510 INFO  [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:50:22,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-03T14:50:22,369 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:50:22,369 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-03T14:50:22,465 INFO  [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T14:50:22,465 INFO  [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T14:50:23,329 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649 after 4002ms
2024-12-03T14:50:23,345 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta after 4001ms
2024-12-03T14:50:23,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:50:23,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741835_1011 (size=393)
2024-12-03T14:50:24,327 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-03T14:50:24,334 DEBUG [RS:1;a5d22df9eca2:44475 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs
2024-12-03T14:50:24,334 INFO  [RS:1;a5d22df9eca2:44475 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C44475%2C1733237382458:(num 1733237382649)
2024-12-03T14:50:24,334 DEBUG [RS:1;a5d22df9eca2:44475 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:24,334 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:50:24,334 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:50:24,335 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-03T14:50:24,335 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:50:24,335 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:50:24,335 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:50:24,336 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:50:24,336 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:50:24,336 INFO  [RS:1;a5d22df9eca2:44475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44475
2024-12-03T14:50:24,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,44475,1733237382458
2024-12-03T14:50:24,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:50:24,338 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:50:24,339 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,44475,1733237382458]
2024-12-03T14:50:24,340 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,44475,1733237382458 already deleted, retry=false
2024-12-03T14:50:24,340 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,44475,1733237382458 expired; onlineServers=1
2024-12-03T14:50:24,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,366 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,376 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,376 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:50:24,440 INFO  [RS:1;a5d22df9eca2:44475 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:50:24,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x100a08ef4800002, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:50:24,440 INFO  [RS:1;a5d22df9eca2:44475 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,44475,1733237382458; zookeeper connection closed.
2024-12-03T14:50:24,440 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ac64957 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ac64957
2024-12-03T14:50:24,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:50:24,501 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:50:24,528 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-03T14:50:24,534 DEBUG [RS:0;a5d22df9eca2:34883 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs
2024-12-03T14:50:24,534 INFO  [RS:0;a5d22df9eca2:34883 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C34883%2C1733237381195.meta:.meta(num 1733237419328)
2024-12-03T14:50:24,535 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,535 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,535 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,535 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,536 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741881_1065 (size=16308)
2024-12-03T14:50:24,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741881_1065 (size=16308)
2024-12-03T14:50:24,542 DEBUG [RS:0;a5d22df9eca2:34883 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/oldWALs
2024-12-03T14:50:24,542 INFO  [RS:0;a5d22df9eca2:34883 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C34883%2C1733237381195:(num 1733237418865)
2024-12-03T14:50:24,542 DEBUG [RS:0;a5d22df9eca2:34883 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:24,542 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:50:24,542 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:50:24,542 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-03T14:50:24,543 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:50:24,543 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:50:24,543 INFO  [RS:0;a5d22df9eca2:34883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34883
2024-12-03T14:50:24,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,34883,1733237381195
2024-12-03T14:50:24,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:50:24,544 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:50:24,545 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,34883,1733237381195]
2024-12-03T14:50:24,545 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,34883,1733237381195 already deleted, retry=false
2024-12-03T14:50:24,545 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,34883,1733237381195 expired; onlineServers=0
2024-12-03T14:50:24,546 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,37049,1733237381157' *****
2024-12-03T14:50:24,546 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T14:50:24,546 INFO  [M:0;a5d22df9eca2:37049 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:50:24,546 INFO  [M:0;a5d22df9eca2:37049 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:50:24,546 DEBUG [M:0;a5d22df9eca2:37049 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T14:50:24,546 DEBUG [M:0;a5d22df9eca2:37049 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T14:50:24,546 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T14:50:24,546 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237381822 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237381822,5,FailOnTimeoutGroup]
2024-12-03T14:50:24,546 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237381822 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237381822,5,FailOnTimeoutGroup]
2024-12-03T14:50:24,546 INFO  [M:0;a5d22df9eca2:37049 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-03T14:50:24,546 INFO  [M:0;a5d22df9eca2:37049 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:50:24,546 DEBUG [M:0;a5d22df9eca2:37049 {}] master.HMaster(1795): Stopping service threads
2024-12-03T14:50:24,547 INFO  [M:0;a5d22df9eca2:37049 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T14:50:24,547 INFO  [M:0;a5d22df9eca2:37049 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:50:24,547 INFO  [M:0;a5d22df9eca2:37049 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T14:50:24,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T14:50:24,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:24,547 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T14:50:24,547 DEBUG [M:0;a5d22df9eca2:37049 {}] zookeeper.ZKUtil(347): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T14:50:24,547 WARN  [M:0;a5d22df9eca2:37049 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T14:50:24,548 INFO  [M:0;a5d22df9eca2:37049 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/.lastflushedseqids
2024-12-03T14:50:24,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741905_1091 (size=130)
2024-12-03T14:50:24,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741905_1091 (size=130)
2024-12-03T14:50:24,553 INFO  [M:0;a5d22df9eca2:37049 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-03T14:50:24,554 INFO  [M:0;a5d22df9eca2:37049 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T14:50:24,554 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:50:24,554 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:24,554 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:24,554 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:50:24,554 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:24,554 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-12-03T14:50:24,569 DEBUG [M:0;a5d22df9eca2:37049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a1a1f603888544838694f4b0fba4e968 is 82, key is hbase:meta,,1/info:regioninfo/1733237382324/Put/seqid=0
2024-12-03T14:50:24,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741906_1092 (size=5672)
2024-12-03T14:50:24,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741906_1092 (size=5672)
2024-12-03T14:50:24,575 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a1a1f603888544838694f4b0fba4e968
2024-12-03T14:50:24,594 DEBUG [M:0;a5d22df9eca2:37049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d545bb489924d56ba01f1cb8c3da3a0 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733237382928/Put/seqid=0
2024-12-03T14:50:24,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741907_1093 (size=6255)
2024-12-03T14:50:24,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741907_1093 (size=6255)
2024-12-03T14:50:24,599 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d545bb489924d56ba01f1cb8c3da3a0
2024-12-03T14:50:24,605 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d545bb489924d56ba01f1cb8c3da3a0
2024-12-03T14:50:24,619 DEBUG [M:0;a5d22df9eca2:37049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8021f3975aee4ba49091d2ba15d6e7cc is 69, key is a5d22df9eca2,34883,1733237381195/rs:state/1733237381863/Put/seqid=0
2024-12-03T14:50:24,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741908_1094 (size=5224)
2024-12-03T14:50:24,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741908_1094 (size=5224)
2024-12-03T14:50:24,625 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8021f3975aee4ba49091d2ba15d6e7cc
2024-12-03T14:50:24,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:50:24,645 INFO  [RS:0;a5d22df9eca2:34883 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:50:24,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34883-0x100a08ef4800001, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:50:24,645 INFO  [RS:0;a5d22df9eca2:34883 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,34883,1733237381195; zookeeper connection closed.
2024-12-03T14:50:24,645 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@770ba0f4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@770ba0f4
2024-12-03T14:50:24,646 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-03T14:50:24,650 DEBUG [M:0;a5d22df9eca2:37049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/435da5f165794c16b87e704931e160b3 is 52, key is load_balancer_on/state:d/1733237382437/Put/seqid=0
2024-12-03T14:50:24,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741909_1095 (size=5056)
2024-12-03T14:50:24,655 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741909_1095 (size=5056)
2024-12-03T14:50:24,655 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/435da5f165794c16b87e704931e160b3
2024-12-03T14:50:24,661 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a1a1f603888544838694f4b0fba4e968 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a1a1f603888544838694f4b0fba4e968
2024-12-03T14:50:24,666 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a1a1f603888544838694f4b0fba4e968, entries=8, sequenceid=60, filesize=5.5 K
2024-12-03T14:50:24,667 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d545bb489924d56ba01f1cb8c3da3a0 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d545bb489924d56ba01f1cb8c3da3a0
2024-12-03T14:50:24,673 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d545bb489924d56ba01f1cb8c3da3a0
2024-12-03T14:50:24,673 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d545bb489924d56ba01f1cb8c3da3a0, entries=6, sequenceid=60, filesize=6.1 K
2024-12-03T14:50:24,674 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8021f3975aee4ba49091d2ba15d6e7cc as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8021f3975aee4ba49091d2ba15d6e7cc
2024-12-03T14:50:24,679 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8021f3975aee4ba49091d2ba15d6e7cc, entries=2, sequenceid=60, filesize=5.1 K
2024-12-03T14:50:24,680 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/435da5f165794c16b87e704931e160b3 as hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/435da5f165794c16b87e704931e160b3
2024-12-03T14:50:24,686 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/435da5f165794c16b87e704931e160b3, entries=1, sequenceid=60, filesize=4.9 K
2024-12-03T14:50:24,687 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 133ms, sequenceid=60, compaction requested=false
2024-12-03T14:50:24,689 INFO  [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:24,689 DEBUG [M:0;a5d22df9eca2:37049 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237424554Disabling compacts and flushes for region at 1733237424554Disabling writes for close at 1733237424554Obtaining lock to block concurrent updates at 1733237424554Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237424554Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1733237424554Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237424555 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237424555Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237424568 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237424568Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237424580 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237424593 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237424593Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237424605 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237424618 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237424618Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733237424631 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733237424649 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733237424649Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10a219b6: reopening flushed file at 1733237424660 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a05c243: reopening flushed file at 1733237424666 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b2605a2: reopening flushed file at 1733237424673 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@462e607c: reopening flushed file at 1733237424679 (+6 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 133ms, sequenceid=60, compaction requested=false at 1733237424687 (+8 ms)Writing region close event to WAL at 1733237424688 (+1 ms)Closed at 1733237424688
2024-12-03T14:50:24,689 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,689 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,689 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,689 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,689 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:24,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741878_1061 (size=1045)
2024-12-03T14:50:24,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741878_1061 (size=1045)
2024-12-03T14:50:24,880 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:50:24,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:24,902 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:25,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:25,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:26,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:26,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:26,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:50:26,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741836_1012 (size=76)
2024-12-03T14:50:27,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:27,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:27,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:50:27,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:50:28,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:28,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:29,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41117 is added to blk_1073741838_1020 (size=3600)
2024-12-03T14:50:29,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:29,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:29,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:50:29,503 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@732cc172 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-133690028-172.17.0.2-1733237380503:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:37547,null,null])
java.net.ConnectException: Call From a5d22df9eca2/172.17.0.2 to localhost:46555 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-03T14:50:29,690 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-03T14:50:29,690 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:50:29,691 INFO  [M:0;a5d22df9eca2:37049 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-03T14:50:29,691 INFO  [M:0;a5d22df9eca2:37049 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37049
2024-12-03T14:50:29,692 INFO  [M:0;a5d22df9eca2:37049 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:50:29,795 INFO  [M:0;a5d22df9eca2:37049 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:50:29,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:50:29,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37049-0x100a08ef4800000, quorum=127.0.0.1:63946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:50:29,800 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@32cdb76b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:29,801 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2c92a539{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:50:29,801 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:50:29,801 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@798caa79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:50:29,801 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7bfce1ef{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,STOPPED}
2024-12-03T14:50:29,804 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:50:29,804 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:50:29,804 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:50:29,804 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-133690028-172.17.0.2-1733237380503 (Datanode Uuid eabbce3a-c616-4ab8-93bd-18ce1ba3c675) service to localhost/127.0.0.1:35825
2024-12-03T14:50:29,805 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data3/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:29,805 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data4/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:29,806 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:50:29,807 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@325d7ee2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:29,808 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15751333{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:50:29,808 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:50:29,808 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c141b19{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:50:29,808 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ab1ed71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,STOPPED}
2024-12-03T14:50:29,809 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:50:29,809 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:50:29,809 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:50:29,809 WARN  [BP-133690028-172.17.0.2-1733237380503 heartbeating to localhost/127.0.0.1:35825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-133690028-172.17.0.2-1733237380503 (Datanode Uuid 78320ed1-34e3-4adc-99ae-95eb16df670a) service to localhost/127.0.0.1:35825
2024-12-03T14:50:29,810 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data9/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:29,810 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/cluster_248a34e6-b33c-fce8-d2bc-367c1f31d50f/data/data10/current/BP-133690028-172.17.0.2-1733237380503 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:29,811 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:50:29,816 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4705e615{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:50:29,816 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d8a9c69{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:50:29,816 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:50:29,816 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16369da1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:50:29,816 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15d1211f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir/,STOPPED}
2024-12-03T14:50:29,823 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-03T14:50:29,850 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-03T14:50:29,858 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=155 (was 78)
Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:35825
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:35825
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:41215
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41215
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35825 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fbe6cbef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fbe6cbef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35825
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35825 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35825
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35825 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35825
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35825
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35825
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35825 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fbe6cbef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=442 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=99 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=4899 (was 5764)
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=155, OpenFileDescriptor=442, MaxFileDescriptor=1048576, SystemLoadAverage=99, ProcessCount=11, AvailableMemoryMB=4899
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.log.dir so I do NOT create it in target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f8597929-05b1-2a22-8b56-d5d257f47397/hadoop.tmp.dir so I do NOT create it in target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295, deleteOnExit=true
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/test.cache.data in system properties and HBase conf
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir in system properties and HBase conf
2024-12-03T14:50:29,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-03T14:50:29,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:29,866 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T14:50:29,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/nfs.dump.dir in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T14:50:29,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T14:50:29,879 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:50:29,924 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:29,928 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:29,929 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:29,929 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:29,929 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:50:29,930 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:29,930 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7bd03e52{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:29,930 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27a49013{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:30,019 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1915705e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir/jetty-localhost-38395-hadoop-hdfs-3_4_1-tests_jar-_-any-5960119212845200750/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:50:30,019 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17b2a9ba{HTTP/1.1, (http/1.1)}{localhost:38395}
2024-12-03T14:50:30,019 INFO  [Time-limited test {}] server.Server(415): Started @153326ms
2024-12-03T14:50:30,030 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:50:30,078 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:30,081 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:30,082 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:30,082 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:30,082 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:50:30,082 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25f63c50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:30,082 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@165d0fad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:30,172 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f0827f7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir/jetty-localhost-36563-hadoop-hdfs-3_4_1-tests_jar-_-any-1849615248955351525/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:30,173 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2744dc92{HTTP/1.1, (http/1.1)}{localhost:36563}
2024-12-03T14:50:30,173 INFO  [Time-limited test {}] server.Server(415): Started @153480ms
2024-12-03T14:50:30,174 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:50:30,199 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:30,201 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:30,202 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:30,202 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:30,202 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:50:30,203 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11255fea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:30,203 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3fd7563{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:30,228 WARN  [Thread-1190 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data1/current/BP-594567723-172.17.0.2-1733237429890/current, will proceed with Du for space computation calculation, 
2024-12-03T14:50:30,228 WARN  [Thread-1191 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data2/current/BP-594567723-172.17.0.2-1733237429890/current, will proceed with Du for space computation calculation, 
2024-12-03T14:50:30,245 WARN  [Thread-1169 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:50:30,247 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc93bdd2e789bdd22 with lease ID 0x8b45f1481965fdff: Processing first storage report for DS-5651eaf4-51ca-47eb-911c-d183237ffff9 from datanode DatanodeRegistration(127.0.0.1:42163, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=36093, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890)
2024-12-03T14:50:30,247 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc93bdd2e789bdd22 with lease ID 0x8b45f1481965fdff: from storage DS-5651eaf4-51ca-47eb-911c-d183237ffff9 node DatanodeRegistration(127.0.0.1:42163, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=36093, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:30,248 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc93bdd2e789bdd22 with lease ID 0x8b45f1481965fdff: Processing first storage report for DS-75b827e6-5f01-4a5a-9057-cee2852a0496 from datanode DatanodeRegistration(127.0.0.1:42163, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=36093, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890)
2024-12-03T14:50:30,248 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc93bdd2e789bdd22 with lease ID 0x8b45f1481965fdff: from storage DS-75b827e6-5f01-4a5a-9057-cee2852a0496 node DatanodeRegistration(127.0.0.1:42163, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=36093, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:30,299 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@140caf6f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir/jetty-localhost-46129-hadoop-hdfs-3_4_1-tests_jar-_-any-14294940268971296352/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:30,300 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da660ce{HTTP/1.1, (http/1.1)}{localhost:46129}
2024-12-03T14:50:30,300 INFO  [Time-limited test {}] server.Server(415): Started @153606ms
2024-12-03T14:50:30,301 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:50:30,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:30,353 WARN  [Thread-1216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data3/current/BP-594567723-172.17.0.2-1733237429890/current, will proceed with Du for space computation calculation, 
2024-12-03T14:50:30,353 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:30,353 WARN  [Thread-1217 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data4/current/BP-594567723-172.17.0.2-1733237429890/current, will proceed with Du for space computation calculation, 
2024-12-03T14:50:30,370 WARN  [Thread-1205 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:50:30,372 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd61c680d535d9d07 with lease ID 0x8b45f1481965fe00: Processing first storage report for DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3 from datanode DatanodeRegistration(127.0.0.1:39809, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=44845, infoSecurePort=0, ipcPort=33199, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890)
2024-12-03T14:50:30,372 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd61c680d535d9d07 with lease ID 0x8b45f1481965fe00: from storage DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3 node DatanodeRegistration(127.0.0.1:39809, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=44845, infoSecurePort=0, ipcPort=33199, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:50:30,372 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd61c680d535d9d07 with lease ID 0x8b45f1481965fe00: Processing first storage report for DS-474cb6ce-35d4-4a29-9569-c904eadca2c3 from datanode DatanodeRegistration(127.0.0.1:39809, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=44845, infoSecurePort=0, ipcPort=33199, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890)
2024-12-03T14:50:30,372 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd61c680d535d9d07 with lease ID 0x8b45f1481965fe00: from storage DS-474cb6ce-35d4-4a29-9569-c904eadca2c3 node DatanodeRegistration(127.0.0.1:39809, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=44845, infoSecurePort=0, ipcPort=33199, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:30,424 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d
2024-12-03T14:50:30,427 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/zookeeper_0, clientPort=53587, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T14:50:30,429 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53587
2024-12-03T14:50:30,429 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:30,432 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:30,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:50:30,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:50:30,443 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3 with version=8
2024-12-03T14:50:30,443 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase-staging
2024-12-03T14:50:30,444 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:50:30,444 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:50:30,444 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:50:30,444 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:50:30,444 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:50:30,444 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:50:30,445 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-03T14:50:30,445 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:50:30,445 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33297
2024-12-03T14:50:30,446 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33297 connecting to ZooKeeper ensemble=127.0.0.1:53587
2024-12-03T14:50:30,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:332970x0, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:50:30,450 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33297-0x100a08fb5090000 connected
2024-12-03T14:50:30,461 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:30,463 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:30,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:50:30,466 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3, hbase.cluster.distributed=false
2024-12-03T14:50:30,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:50:30,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33297
2024-12-03T14:50:30,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33297
2024-12-03T14:50:30,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33297
2024-12-03T14:50:30,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33297
2024-12-03T14:50:30,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33297
2024-12-03T14:50:30,487 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:50:30,487 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:50:30,487 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:50:30,487 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:50:30,487 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:50:30,487 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:50:30,487 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:50:30,488 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:50:30,488 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41595
2024-12-03T14:50:30,489 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41595 connecting to ZooKeeper ensemble=127.0.0.1:53587
2024-12-03T14:50:30,490 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:30,491 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:30,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415950x0, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:50:30,495 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41595-0x100a08fb5090001 connected
2024-12-03T14:50:30,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:50:30,495 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:50:30,495 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:50:30,496 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T14:50:30,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:50:30,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41595
2024-12-03T14:50:30,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41595
2024-12-03T14:50:30,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41595
2024-12-03T14:50:30,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41595
2024-12-03T14:50:30,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41595
2024-12-03T14:50:30,507 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:33297
2024-12-03T14:50:30,508 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:30,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:50:30,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:50:30,509 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:30,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T14:50:30,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,510 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:50:30,511 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,33297,1733237430444 from backup master directory
2024-12-03T14:50:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:50:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:50:30,511 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:50:30,512 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:30,516 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/hbase.id] with ID: 43ae125b-451e-4f53-95df-dbe35c088fd3
2024-12-03T14:50:30,516 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/.tmp/hbase.id
2024-12-03T14:50:30,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:50:30,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:50:30,526 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/.tmp/hbase.id]:[hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/hbase.id]
2024-12-03T14:50:30,537 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:30,537 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-03T14:50:30,538 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-03T14:50:30,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:50:30,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:50:30,547 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:50:30,548 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T14:50:30,548 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:50:30,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:50:30,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:50:30,555 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store
2024-12-03T14:50:30,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:50:30,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:50:30,563 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:50:30,563 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:50:30,563 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:30,563 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:30,563 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:50:30,563 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:30,563 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:50:30,563 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237430563Disabling compacts and flushes for region at 1733237430563Disabling writes for close at 1733237430563Writing region close event to WAL at 1733237430563Closed at 1733237430563
2024-12-03T14:50:30,564 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/.initializing
2024-12-03T14:50:30,564 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:30,566 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C33297%2C1733237430444, suffix=, logDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444, archiveDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/oldWALs, maxLogs=10
2024-12-03T14:50:30,567 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C33297%2C1733237430444.1733237430567
2024-12-03T14:50:30,571 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567
2024-12-03T14:50:30,571 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36093:36093),(127.0.0.1/127.0.0.1:44845:44845)]
2024-12-03T14:50:30,572 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:50:30,572 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:50:30,572 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,572 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,576 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:50:30,578 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T14:50:30,578 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,578 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:30,578 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:50:30,579 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T14:50:30,579 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,580 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:50:30,580 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:50:30,581 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T14:50:30,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,581 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:50:30,581 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:50:30,582 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T14:50:30,583 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,583 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:50:30,583 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,584 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,584 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,585 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,585 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,585 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T14:50:30,586 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:50:30,588 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:50:30,589 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=768319, jitterRate=-0.023032575845718384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T14:50:30,589 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733237430572Initializing all the Stores at 1733237430573 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237430573Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237430576 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237430576Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237430576Cleaning up temporary data from old regions at 1733237430585 (+9 ms)Region opened successfully at 1733237430589 (+4 ms)
2024-12-03T14:50:30,590 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T14:50:30,593 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558487a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:50:30,594 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-03T14:50:30,594 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T14:50:30,594 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T14:50:30,594 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T14:50:30,594 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T14:50:30,595 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-03T14:50:30,595 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T14:50:30,599 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T14:50:30,600 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T14:50:30,601 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-03T14:50:30,601 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T14:50:30,601 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T14:50:30,602 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-03T14:50:30,602 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T14:50:30,603 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T14:50:30,604 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-03T14:50:30,605 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T14:50:30,606 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T14:50:30,608 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T14:50:30,609 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T14:50:30,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:50:30,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:50:30,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,610 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,33297,1733237430444, sessionid=0x100a08fb5090000, setting cluster-up flag (Was=false)
2024-12-03T14:50:30,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,615 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T14:50:30,615 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:30,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:30,620 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T14:50:30,620 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:30,621 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-03T14:50:30,623 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-03T14:50:30,623 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-03T14:50:30,623 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T14:50:30,623 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,33297,1733237430444
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:50:30,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237460630
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T14:50:30,630 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:50:30,630 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,630 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-03T14:50:30,631 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T14:50:30,631 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T14:50:30,631 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T14:50:30,631 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T14:50:30,631 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T14:50:30,631 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,632 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T14:50:30,633 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237430631,5,FailOnTimeoutGroup]
2024-12-03T14:50:30,634 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237430633,5,FailOnTimeoutGroup]
2024-12-03T14:50:30,634 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,634 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T14:50:30,634 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,634 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:50:30,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:50:30,640 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-03T14:50:30,640 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3
2024-12-03T14:50:30,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:50:30,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:50:30,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:50:30,653 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:50:30,654 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:50:30,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,655 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:30,655 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:50:30,656 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:50:30,656 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,657 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:30,657 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:50:30,658 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:50:30,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,658 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:30,658 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:50:30,659 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:50:30,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:30,660 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:30,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:50:30,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740
2024-12-03T14:50:30,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740
2024-12-03T14:50:30,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:50:30,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:50:30,662 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:50:30,669 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:50:30,671 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:50:30,671 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873887, jitterRate=0.11120596528053284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:50:30,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733237430650Initializing all the Stores at 1733237430651 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237430651Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237430653 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237430653Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237430653Cleaning up temporary data from old regions at 1733237430662 (+9 ms)Region opened successfully at 1733237430672 (+10 ms)
2024-12-03T14:50:30,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:50:30,672 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:50:30,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:50:30,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:50:30,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:50:30,673 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:50:30,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237430672Disabling compacts and flushes for region at 1733237430672Disabling writes for close at 1733237430672Writing region close event to WAL at 1733237430673 (+1 ms)Closed at 1733237430673
2024-12-03T14:50:30,674 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:50:30,674 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-03T14:50:30,674 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T14:50:30,676 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:50:30,677 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T14:50:30,700 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(746): ClusterId : 43ae125b-451e-4f53-95df-dbe35c088fd3
2024-12-03T14:50:30,700 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:50:30,702 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:50:30,702 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:50:30,704 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:50:30,704 DEBUG [RS:0;a5d22df9eca2:41595 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@507eceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:50:30,713 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:41595
2024-12-03T14:50:30,714 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:50:30,714 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:50:30,714 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:50:30,714 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,33297,1733237430444 with port=41595, startcode=1733237430487
2024-12-03T14:50:30,715 DEBUG [RS:0;a5d22df9eca2:41595 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:50:30,717 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55105, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:50:30,717 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33297 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:30,717 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33297 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:30,719 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3
2024-12-03T14:50:30,719 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43907
2024-12-03T14:50:30,719 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:50:30,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:50:30,721 DEBUG [RS:0;a5d22df9eca2:41595 {}] zookeeper.ZKUtil(111): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:30,721 WARN  [RS:0;a5d22df9eca2:41595 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:50:30,721 INFO  [RS:0;a5d22df9eca2:41595 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:50:30,721 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:30,721 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,41595,1733237430487]
2024-12-03T14:50:30,724 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:50:30,725 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:50:30,726 INFO  [RS:0;a5d22df9eca2:41595 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:50:30,726 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,727 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:50:30,728 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:50:30,728 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:50:30,728 DEBUG [RS:0;a5d22df9eca2:41595 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:50:30,729 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,729 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,729 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,729 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,729 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,730 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,41595,1733237430487-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:50:30,744 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:50:30,744 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,41595,1733237430487-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,744 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,744 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.Replication(171): a5d22df9eca2,41595,1733237430487 started
2024-12-03T14:50:30,755 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:30,755 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,41595,1733237430487, RpcServer on a5d22df9eca2/172.17.0.2:41595, sessionid=0x100a08fb5090001
2024-12-03T14:50:30,755 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:50:30,755 DEBUG [RS:0;a5d22df9eca2:41595 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:30,755 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,41595,1733237430487'
2024-12-03T14:50:30,755 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:50:30,756 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:50:30,756 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:50:30,756 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:50:30,757 DEBUG [RS:0;a5d22df9eca2:41595 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:30,757 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,41595,1733237430487'
2024-12-03T14:50:30,757 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:50:30,757 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:50:30,757 DEBUG [RS:0;a5d22df9eca2:41595 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:50:30,757 INFO  [RS:0;a5d22df9eca2:41595 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:50:30,757 INFO  [RS:0;a5d22df9eca2:41595 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:50:30,827 WARN  [a5d22df9eca2:33297 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-03T14:50:30,860 INFO  [RS:0;a5d22df9eca2:41595 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C41595%2C1733237430487, suffix=, logDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487, archiveDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/oldWALs, maxLogs=32
2024-12-03T14:50:30,862 INFO  [RS:0;a5d22df9eca2:41595 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:30,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:30,870 INFO  [RS:0;a5d22df9eca2:41595 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:30,872 DEBUG [RS:0;a5d22df9eca2:41595 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44845:44845),(127.0.0.1/127.0.0.1:36093:36093)]
2024-12-03T14:50:31,077 DEBUG [a5d22df9eca2:33297 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T14:50:31,079 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:31,083 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,41595,1733237430487, state=OPENING
2024-12-03T14:50:31,085 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T14:50:31,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:31,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:31,086 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:50:31,087 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:50:31,087 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:50:31,087 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,41595,1733237430487}]
2024-12-03T14:50:31,241 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T14:50:31,246 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54075, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T14:50:31,253 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-03T14:50:31,253 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:50:31,255 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C41595%2C1733237430487.meta, suffix=.meta, logDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487, archiveDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/oldWALs, maxLogs=32
2024-12-03T14:50:31,255 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta
2024-12-03T14:50:31,260 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta
2024-12-03T14:50:31,261 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44845:44845),(127.0.0.1/127.0.0.1:36093:36093)]
2024-12-03T14:50:31,261 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:50:31,262 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T14:50:31,262 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T14:50:31,262 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T14:50:31,262 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T14:50:31,262 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:50:31,262 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-03T14:50:31,262 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-03T14:50:31,263 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:50:31,264 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:50:31,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:31,265 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:31,265 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:50:31,266 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:50:31,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:31,267 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:31,267 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:50:31,268 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:50:31,268 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:31,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:31,269 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:50:31,269 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:50:31,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:31,270 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:50:31,270 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:50:31,271 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740
2024-12-03T14:50:31,272 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740
2024-12-03T14:50:31,273 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:50:31,273 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:50:31,274 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:50:31,275 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:50:31,276 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=809529, jitterRate=0.029370442032814026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:50:31,276 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-03T14:50:31,276 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733237431262Writing region info on filesystem at 1733237431262Initializing all the Stores at 1733237431263 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237431263Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237431263Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237431263Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237431263Cleaning up temporary data from old regions at 1733237431273 (+10 ms)Running coprocessor post-open hooks at 1733237431276 (+3 ms)Region opened successfully at 1733237431276
2024-12-03T14:50:31,277 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237431241
2024-12-03T14:50:31,279 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T14:50:31,279 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-03T14:50:31,280 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:31,281 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,41595,1733237430487, state=OPEN
2024-12-03T14:50:31,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:50:31,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:50:31,284 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:31,284 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:50:31,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:50:31,287 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T14:50:31,287 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,41595,1733237430487 in 197 msec
2024-12-03T14:50:31,289 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T14:50:31,289 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec
2024-12-03T14:50:31,290 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:50:31,290 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-03T14:50:31,291 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:50:31,292 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,41595,1733237430487, seqNum=-1]
2024-12-03T14:50:31,292 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:50:31,293 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57003, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:50:31,299 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 675 msec
2024-12-03T14:50:31,299 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237431299, completionTime=-1
2024-12-03T14:50:31,299 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T14:50:31,299 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-03T14:50:31,301 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-03T14:50:31,301 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237491301
2024-12-03T14:50:31,301 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237551301
2024-12-03T14:50:31,301 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-03T14:50:31,301 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,33297,1733237430444-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:31,301 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,33297,1733237430444-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:31,302 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,33297,1733237430444-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:31,302 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:33297, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:31,302 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:31,302 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:31,304 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.794sec
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,33297,1733237430444-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:50:31,306 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,33297,1733237430444-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T14:50:31,309 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-03T14:50:31,309 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T14:50:31,309 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,33297,1733237430444-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:50:31,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:31,353 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:31,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40375d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:50:31,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,33297,-1 for getting cluster id
2024-12-03T14:50:31,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-03T14:50:31,403 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '43ae125b-451e-4f53-95df-dbe35c088fd3'
2024-12-03T14:50:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-03T14:50:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "43ae125b-451e-4f53-95df-dbe35c088fd3"

2024-12-03T14:50:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbd97ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:50:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,33297,-1]
2024-12-03T14:50:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-03T14:50:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:31,408 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-03T14:50:31,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6144f613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:50:31,409 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:50:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,41595,1733237430487, seqNum=-1]
2024-12-03T14:50:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:50:31,413 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:50:31,416 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:31,416 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:50:31,420 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-03T14:50:31,420 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-03T14:50:31,421 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-03T14:50:31,421 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-03T14:50:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@af8923
2024-12-03T14:50:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T14:50:31,424 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T14:50:31,425 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33297 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T14:50:31,425 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33297 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T14:50:31,425 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33297 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:50:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33297 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T14:50:31,429 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T14:50:31,429 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:31,429 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33297 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-03T14:50:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:50:31,431 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T14:50:31,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741835_1011 (size=395)
2024-12-03T14:50:31,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741835_1011 (size=395)
2024-12-03T14:50:31,439 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab0d4a856efb6f578508d0b3f42434e0, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3
2024-12-03T14:50:31,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42163 is added to blk_1073741836_1012 (size=78)
2024-12-03T14:50:31,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39809 is added to blk_1073741836_1012 (size=78)
2024-12-03T14:50:31,447 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:50:31,447 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing ab0d4a856efb6f578508d0b3f42434e0, disabling compactions & flushes
2024-12-03T14:50:31,447 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:31,447 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:31,447 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0. after waiting 0 ms
2024-12-03T14:50:31,447 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:31,447 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:31,447 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab0d4a856efb6f578508d0b3f42434e0:
Waiting for close lock at 1733237431447Disabling compacts and flushes for region at 1733237431447Disabling writes for close at 1733237431447Writing region close event to WAL at 1733237431447Closed at 1733237431447
2024-12-03T14:50:31,449 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T14:50:31,450 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733237431449"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237431449"}]},"ts":"1733237431449"}
2024-12-03T14:50:31,453 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-03T14:50:31,454 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T14:50:31,454 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237431454"}]},"ts":"1733237431454"}
2024-12-03T14:50:31,457 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-03T14:50:31,457 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ab0d4a856efb6f578508d0b3f42434e0, ASSIGN}]
2024-12-03T14:50:31,459 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ab0d4a856efb6f578508d0b3f42434e0, ASSIGN
2024-12-03T14:50:31,460 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ab0d4a856efb6f578508d0b3f42434e0, ASSIGN; state=OFFLINE, location=a5d22df9eca2,41595,1733237430487; forceNewPlan=false, retain=false
2024-12-03T14:50:31,611 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ab0d4a856efb6f578508d0b3f42434e0, regionState=OPENING, regionLocation=a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:31,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ab0d4a856efb6f578508d0b3f42434e0, ASSIGN because future has completed
2024-12-03T14:50:31,619 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab0d4a856efb6f578508d0b3f42434e0, server=a5d22df9eca2,41595,1733237430487}]
2024-12-03T14:50:31,782 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:31,782 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ab0d4a856efb6f578508d0b3f42434e0, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:50:31,783 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,783 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:50:31,783 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,783 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,785 INFO  [StoreOpener-ab0d4a856efb6f578508d0b3f42434e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ab0d4a856efb6f578508d0b3f42434e0 
2024-12-03T14:50:31,787 INFO  [StoreOpener-ab0d4a856efb6f578508d0b3f42434e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab0d4a856efb6f578508d0b3f42434e0 columnFamilyName info
2024-12-03T14:50:31,787 DEBUG [StoreOpener-ab0d4a856efb6f578508d0b3f42434e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:50:31,787 INFO  [StoreOpener-ab0d4a856efb6f578508d0b3f42434e0-1 {}] regionserver.HStore(327): Store=ab0d4a856efb6f578508d0b3f42434e0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:50:31,788 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,788 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,789 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,789 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,790 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,792 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,795 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:50:31,796 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ab0d4a856efb6f578508d0b3f42434e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=786194, jitterRate=-3.038942813873291E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T14:50:31,796 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:31,797 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ab0d4a856efb6f578508d0b3f42434e0:
Running coprocessor pre-open hook at 1733237431783Writing region info on filesystem at 1733237431783Initializing all the Stores at 1733237431785 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237431785Cleaning up temporary data from old regions at 1733237431790 (+5 ms)Running coprocessor post-open hooks at 1733237431796 (+6 ms)Region opened successfully at 1733237431797 (+1 ms)
2024-12-03T14:50:31,798 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0., pid=6, masterSystemTime=1733237431774
2024-12-03T14:50:31,801 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:31,801 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:31,802 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ab0d4a856efb6f578508d0b3f42434e0, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:31,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab0d4a856efb6f578508d0b3f42434e0, server=a5d22df9eca2,41595,1733237430487 because future has completed
2024-12-03T14:50:31,810 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T14:50:31,810 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ab0d4a856efb6f578508d0b3f42434e0, server=a5d22df9eca2,41595,1733237430487 in 187 msec
2024-12-03T14:50:31,813 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T14:50:31,813 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ab0d4a856efb6f578508d0b3f42434e0, ASSIGN in 353 msec
2024-12-03T14:50:31,815 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T14:50:31,815 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237431815"}]},"ts":"1733237431815"}
2024-12-03T14:50:31,817 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-03T14:50:31,819 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T14:50:31,822 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 393 msec
2024-12-03T14:50:31,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:32,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:32,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:32,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-03T14:50:32,366 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-03T14:50:32,368 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T14:50:32,368 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-03T14:50:32,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:50:32,370 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-03T14:50:32,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:33,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:33,355 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:33,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:34,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:34,357 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:34,872 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:35,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:35,357 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:35,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:36,279 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,279 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,280 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,280 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,280 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,280 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,282 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,282 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,283 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,284 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:36,358 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:36,789 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:50:36,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,813 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,813 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,813 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,814 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:50:36,819 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T14:50:36,819 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-03T14:50:36,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:37,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:37,359 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:37,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:38,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:38,360 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:38,876 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:39,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:39,361 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:39,877 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:40,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:40,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:40,878 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:41,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:41,364 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:50:41,504 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-03T14:50:41,504 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-03T14:50:41,511 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T14:50:41,511 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0., hostname=a5d22df9eca2,41595,1733237430487, seqNum=2]
2024-12-03T14:50:41,880 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:42,353 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:42,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:42,881 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:43,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:43,366 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:43,522 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:43,522 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:43,522 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:43,523 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:39809,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:43,523 WARN  [DataStreamer for file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 block BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39809,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK], DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39809,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]) is bad.
2024-12-03T14:50:43,523 WARN  [DataStreamer for file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta block BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39809,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK], DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39809,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]) is bad.
2024-12-03T14:50:43,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:60254 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39809:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60254 dst: /127.0.0.1:39809
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:60240 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39809:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60240 dst: /127.0.0.1:39809
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,523 WARN  [DataStreamer for file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 block BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK], DatanodeInfoWithStorage[127.0.0.1:39809,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39809,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]) is bad.
2024-12-03T14:50:43,523 WARN  [PacketResponder: BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39809] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:48072 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48072 dst: /127.0.0.1:42163
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-818849242_22 at /127.0.0.1:48042 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48042 dst: /127.0.0.1:42163
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:48080 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48080 dst: /127.0.0.1:42163
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-818849242_22 at /127.0.0.1:60226 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39809:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60226 dst: /127.0.0.1:39809
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,526 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@140caf6f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:43,527 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1da660ce{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:50:43,527 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:50:43,527 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3fd7563{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:50:43,527 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11255fea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,STOPPED}
2024-12-03T14:50:43,529 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:50:43,529 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:50:43,529 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-594567723-172.17.0.2-1733237429890 (Datanode Uuid 31376101-3aac-42b1-afb1-79f3c7db9fdd) service to localhost/127.0.0.1:43907
2024-12-03T14:50:43,529 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:50:43,529 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data3/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:43,530 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data4/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:43,530 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:50:43,537 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:43,540 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:43,541 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:43,541 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:43,541 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:50:43,541 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5960bf29{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:43,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c07ac8c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:43,633 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6a4031cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir/jetty-localhost-34221-hadoop-hdfs-3_4_1-tests_jar-_-any-9785436167085553641/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:43,633 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@126bd190{HTTP/1.1, (http/1.1)}{localhost:34221}
2024-12-03T14:50:43,633 INFO  [Time-limited test {}] server.Server(415): Started @166940ms
2024-12-03T14:50:43,634 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:50:43,653 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:43,653 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:43,653 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:43,654 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:50350 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50350 dst: /127.0.0.1:42163
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,654 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:50352 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50352 dst: /127.0.0.1:42163
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,654 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-818849242_22 at /127.0.0.1:50376 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50376 dst: /127.0.0.1:42163
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:43,658 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f0827f7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:43,658 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2744dc92{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:50:43,658 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:50:43,658 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@165d0fad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:50:43,658 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25f63c50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,STOPPED}
2024-12-03T14:50:43,661 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:50:43,662 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:50:43,662 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:50:43,662 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-594567723-172.17.0.2-1733237429890 (Datanode Uuid 7dbe9ada-1e0f-4f81-8c47-1dc93f83a813) service to localhost/127.0.0.1:43907
2024-12-03T14:50:43,662 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data1/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:43,662 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data2/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:43,663 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:50:43,669 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:43,673 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:43,674 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:43,674 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:43,674 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:50:43,675 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@176f5faa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:43,675 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3237c5f1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:43,707 WARN  [Thread-1340 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:50:43,709 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf60b99038bc53377 with lease ID 0x8b45f1481965fe01: from storage DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3 node DatanodeRegistration(127.0.0.1:38517, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=42201, infoSecurePort=0, ipcPort=37493, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:43,709 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf60b99038bc53377 with lease ID 0x8b45f1481965fe01: from storage DS-474cb6ce-35d4-4a29-9569-c904eadca2c3 node DatanodeRegistration(127.0.0.1:38517, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=42201, infoSecurePort=0, ipcPort=37493, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:43,773 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@358d2587{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir/jetty-localhost-39823-hadoop-hdfs-3_4_1-tests_jar-_-any-12626790182634212396/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:43,773 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e49578b{HTTP/1.1, (http/1.1)}{localhost:39823}
2024-12-03T14:50:43,773 INFO  [Time-limited test {}] server.Server(415): Started @167080ms
2024-12-03T14:50:43,774 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:50:43,861 WARN  [Thread-1371 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:50:43,864 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7ee84bda30dd470 with lease ID 0x8b45f1481965fe02: from storage DS-5651eaf4-51ca-47eb-911c-d183237ffff9 node DatanodeRegistration(127.0.0.1:42631, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=43463, infoSecurePort=0, ipcPort=33415, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:43,864 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7ee84bda30dd470 with lease ID 0x8b45f1481965fe02: from storage DS-75b827e6-5f01-4a5a-9057-cee2852a0496 node DatanodeRegistration(127.0.0.1:42631, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=43463, infoSecurePort=0, ipcPort=33415, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:43,883 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:44,355 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:44,367 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:44,820 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-03T14:50:44,825 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-03T14:50:44,828 ERROR [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:44,828 WARN  [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:44,828 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C41595%2C1733237430487:(num 1733237430861) roll requested
2024-12-03T14:50:44,828 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:44,836 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 newFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:44,837 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:44,837 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:44,837 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:44,837 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:44,837 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:44,837 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:44,838 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:44,838 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:44,838 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:44,839 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43463:43463),(127.0.0.1/127.0.0.1:42201:42201)]
2024-12-03T14:50:44,839 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 is not closed yet, will try archiving it next time
2024-12-03T14:50:44,839 WARN  [IPC Server handler 3 on default port 43907 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-12-03T14:50:44,839 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 after 1ms
2024-12-03T14:50:44,883 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:45,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:45,368 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:45,710 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T14:50:45,884 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:46,357 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:46,369 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:46,843 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-03T14:50:46,885 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:47,358 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:47,370 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:47,886 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:48,360 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:48,371 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:48,841 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 after 4003ms
2024-12-03T14:50:48,848 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:48,849 WARN  [DataStreamer for file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 block BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42631,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK], DatanodeInfoWithStorage[127.0.0.1:38517,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42631,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]) is bad.
2024-12-03T14:50:48,850 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:36920 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:42631:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36920 dst: /127.0.0.1:42631
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:48,850 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:35042 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38517:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35042 dst: /127.0.0.1:38517
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:48,853 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@358d2587{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:48,854 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e49578b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:50:48,854 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:50:48,854 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3237c5f1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:50:48,854 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@176f5faa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,STOPPED}
2024-12-03T14:50:48,856 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:50:48,856 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:50:48,856 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:50:48,856 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-594567723-172.17.0.2-1733237429890 (Datanode Uuid 7dbe9ada-1e0f-4f81-8c47-1dc93f83a813) service to localhost/127.0.0.1:43907
2024-12-03T14:50:48,856 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data1/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:48,857 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data2/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:48,857 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:50:48,870 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:48,874 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:48,875 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:48,875 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:48,875 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:50:48,875 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@718ea2f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:48,876 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@167fd01b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:48,887 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:48,965 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ca8564b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir/jetty-localhost-38343-hadoop-hdfs-3_4_1-tests_jar-_-any-14815999284831263725/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:48,965 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@fa662a{HTTP/1.1, (http/1.1)}{localhost:38343}
2024-12-03T14:50:48,965 INFO  [Time-limited test {}] server.Server(415): Started @172272ms
2024-12-03T14:50:48,966 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:50:48,980 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:48,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-312462901_22 at /127.0.0.1:35062 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38517:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35062 dst: /127.0.0.1:38517
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:48,985 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6a4031cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:48,986 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@126bd190{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:50:48,986 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:50:48,986 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c07ac8c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:50:48,986 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5960bf29{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,STOPPED}
2024-12-03T14:50:48,987 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:50:48,987 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:50:48,987 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:50:48,987 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-594567723-172.17.0.2-1733237429890 (Datanode Uuid 31376101-3aac-42b1-afb1-79f3c7db9fdd) service to localhost/127.0.0.1:43907
2024-12-03T14:50:48,987 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data3/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:48,987 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data4/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:50:48,987 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:50:48,999 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:50:49,003 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:50:49,004 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:50:49,004 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:50:49,004 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:50:49,004 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@760c54cf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:50:49,005 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2026736d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:50:49,036 WARN  [Thread-1414 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:50:49,038 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1aed265b9836d55a with lease ID 0x8b45f1481965fe03: from storage DS-5651eaf4-51ca-47eb-911c-d183237ffff9 node DatanodeRegistration(127.0.0.1:36479, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=35093, infoSecurePort=0, ipcPort=33655, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T14:50:49,038 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1aed265b9836d55a with lease ID 0x8b45f1481965fe03: from storage DS-75b827e6-5f01-4a5a-9057-cee2852a0496 node DatanodeRegistration(127.0.0.1:36479, datanodeUuid=7dbe9ada-1e0f-4f81-8c47-1dc93f83a813, infoPort=35093, infoSecurePort=0, ipcPort=33655, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:49,099 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@528eeea6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/java.io.tmpdir/jetty-localhost-40033-hadoop-hdfs-3_4_1-tests_jar-_-any-6428746466592739083/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:50:49,099 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20466b6{HTTP/1.1, (http/1.1)}{localhost:40033}
2024-12-03T14:50:49,099 INFO  [Time-limited test {}] server.Server(415): Started @172406ms
2024-12-03T14:50:49,100 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:50:49,169 WARN  [Thread-1445 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:50:49,171 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2f39e9b0cd7b2 with lease ID 0x8b45f1481965fe04: from storage DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3 node DatanodeRegistration(127.0.0.1:33337, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=42517, infoSecurePort=0, ipcPort=35185, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:49,171 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2f39e9b0cd7b2 with lease ID 0x8b45f1481965fe04: from storage DS-474cb6ce-35d4-4a29-9569-c904eadca2c3 node DatanodeRegistration(127.0.0.1:33337, datanodeUuid=31376101-3aac-42b1-afb1-79f3c7db9fdd, infoPort=42517, infoSecurePort=0, ipcPort=35185, storageInfo=lv=-57;cid=testClusterID;nsid=1655931651;c=1733237429890), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:50:49,361 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:49,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:49,888 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:50,117 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-03T14:50:50,120 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-03T14:50:50,123 ERROR [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38517,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:50,123 WARN  [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38517,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:50,123 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C41595%2C1733237430487:(num 1733237444828) roll requested
2024-12-03T14:50:50,124 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C41595%2C1733237430487.1733237450123
2024-12-03T14:50:50,132 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 newFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123
2024-12-03T14:50:50,132 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:50,132 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:50,132 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:50,132 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:50,132 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:50,133 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123
2024-12-03T14:50:50,133 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38517,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:50,133 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38517,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:50,133 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:50,134 WARN  [IPC Server handler 4 on default port 43907 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-03T14:50:50,134 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 after 1ms
2024-12-03T14:50:50,134 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35093:35093),(127.0.0.1/127.0.0.1:42517:42517)]
2024-12-03T14:50:50,134 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 is not closed yet, will try archiving it next time
2024-12-03T14:50:50,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:50,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:50,890 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:51,363 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:51,374 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:51,891 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:52,038 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T14:50:52,136 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:52,141 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123 newFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:52,142 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:52,142 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:52,142 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:52,142 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:52,142 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:52,142 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:52,145 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42517:42517),(127.0.0.1/127.0.0.1:35093:35093)]
2024-12-03T14:50:52,145 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 is not closed yet, will try archiving it next time
2024-12-03T14:50:52,145 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123 is not closed yet, will try archiving it next time
2024-12-03T14:50:52,145 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:52,145 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:52,146 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861 after 1ms
2024-12-03T14:50:52,146 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:52,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741838_1019 (size=1264)
2024-12-03T14:50:52,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741838_1019 (size=1264)
2024-12-03T14:50:52,146 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 is not closed yet, will try archiving it next time
2024-12-03T14:50:52,154 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733237431797/Put/vlen=218/seqid=0]
2024-12-03T14:50:52,154 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733237441520/Put/vlen=1045/seqid=0]
2024-12-03T14:50:52,154 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237430861
2024-12-03T14:50:52,154 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:52,154 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:52,155 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 after 1ms
2024-12-03T14:50:52,155 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:52,158 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733237444827/Put/vlen=1045/seqid=0]
2024-12-03T14:50:52,158 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733237446845/Put/vlen=1045/seqid=0]
2024-12-03T14:50:52,158 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828
2024-12-03T14:50:52,158 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123
2024-12-03T14:50:52,158 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123
2024-12-03T14:50:52,159 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123 after 1ms
2024-12-03T14:50:52,159 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237450123
2024-12-03T14:50:52,162 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733237450122/Put/vlen=1045/seqid=0]
2024-12-03T14:50:52,162 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:52,162 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:52,162 WARN  [IPC Server handler 2 on default port 43907 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-03T14:50:52,162 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 after 0ms
2024-12-03T14:50:52,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:52,375 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:52,892 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:53,045 WARN  [ResponseProcessor for block BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:53,045 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-818849242_22 at /127.0.0.1:56168 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:33337:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56168 dst: /127.0.0.1:33337
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33337 remote=/127.0.0.1:56168]. Total timeout mills is 60000, 59096 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:53,046 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-818849242_22 at /127.0.0.1:32768 [Receiving block BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:36479:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:32768 dst: /127.0.0.1:36479
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:50:53,046 WARN  [DataStreamer for file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 block BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33337,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK], DatanodeInfoWithStorage[127.0.0.1:36479,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33337,DS-71f21517-805b-4f3b-8ba9-2f6f57fb67b3,DISK]) is bad.
2024-12-03T14:50:53,049 WARN  [DataStreamer for file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 block BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:53,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741839_1022 (size=85)
2024-12-03T14:50:53,366 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:53,376 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:53,893 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:54,135 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237444828 after 4002ms
2024-12-03T14:50:54,367 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:54,377 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:54,894 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:55,368 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:55,378 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:55,894 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:56,163 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 after 4001ms
2024-12-03T14:50:56,163 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:56,166 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:56,167 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-03T14:50:56,167 ERROR [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,167 WARN  [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,167 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C41595%2C1733237430487.meta:.meta(num 1733237431255) roll requested
2024-12-03T14:50:56,168 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C41595%2C1733237430487.meta.1733237456168.meta
2024-12-03T14:50:56,172 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,173 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,173 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,173 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,173 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,173 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237456168.meta
2024-12-03T14:50:56,173 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,173 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,173 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta
2024-12-03T14:50:56,174 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35093:35093),(127.0.0.1/127.0.0.1:42517:42517)]
2024-12-03T14:50:56,174 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta is not closed yet, will try archiving it next time
2024-12-03T14:50:56,174 WARN  [IPC Server handler 0 on default port 43907 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013
2024-12-03T14:50:56,174 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta after 1ms
2024-12-03T14:50:56,190 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/info/925b5fd23c2a47e4be9db52562771ad0 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0./info:regioninfo/1733237431802/Put/seqid=0
2024-12-03T14:50:56,195 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741841_1025 (size=7125)
2024-12-03T14:50:56,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741841_1025 (size=7125)
2024-12-03T14:50:56,196 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/info/925b5fd23c2a47e4be9db52562771ad0
2024-12-03T14:50:56,218 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/ns/b724370432d74442b353176dcc26d3cb is 43, key is default/ns:d/1733237431293/Put/seqid=0
2024-12-03T14:50:56,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741842_1026 (size=5153)
2024-12-03T14:50:56,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741842_1026 (size=5153)
2024-12-03T14:50:56,224 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/ns/b724370432d74442b353176dcc26d3cb
2024-12-03T14:50:56,244 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/table/f9fe1783d2784b6ebecfa3b7ff5c0337 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733237431815/Put/seqid=0
2024-12-03T14:50:56,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741843_1027 (size=5438)
2024-12-03T14:50:56,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741843_1027 (size=5438)
2024-12-03T14:50:56,250 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/table/f9fe1783d2784b6ebecfa3b7ff5c0337
2024-12-03T14:50:56,259 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/info/925b5fd23c2a47e4be9db52562771ad0 as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/info/925b5fd23c2a47e4be9db52562771ad0
2024-12-03T14:50:56,268 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/info/925b5fd23c2a47e4be9db52562771ad0, entries=10, sequenceid=11, filesize=7.0 K
2024-12-03T14:50:56,269 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/ns/b724370432d74442b353176dcc26d3cb as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/ns/b724370432d74442b353176dcc26d3cb
2024-12-03T14:50:56,276 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/ns/b724370432d74442b353176dcc26d3cb, entries=2, sequenceid=11, filesize=5.0 K
2024-12-03T14:50:56,277 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/.tmp/table/f9fe1783d2784b6ebecfa3b7ff5c0337 as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/table/f9fe1783d2784b6ebecfa3b7ff5c0337
2024-12-03T14:50:56,284 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/table/f9fe1783d2784b6ebecfa3b7ff5c0337, entries=2, sequenceid=11, filesize=5.3 K
2024-12-03T14:50:56,286 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 119ms, sequenceid=11, compaction requested=false
2024-12-03T14:50:56,286 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-03T14:50:56,286 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing ab0d4a856efb6f578508d0b3f42434e0 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-03T14:50:56,286 ERROR [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,287 WARN  [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3-prefix:a5d22df9eca2,41595,1733237430487 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,287 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C41595%2C1733237430487:(num 1733237452136) roll requested
2024-12-03T14:50:56,287 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C41595%2C1733237430487.1733237456287
2024-12-03T14:50:56,293 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 newFile=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237456287
2024-12-03T14:50:56,294 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,294 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,294 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,294 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,294 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,294 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237456287
2024-12-03T14:50:56,294 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,295 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-594567723-172.17.0.2-1733237429890:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:50:56,295 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:56,295 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 after 0ms
2024-12-03T14:50:56,295 DEBUG [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42517:42517),(127.0.0.1/127.0.0.1:35093:35093)]
2024-12-03T14:50:56,296 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.1733237452136 to hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/oldWALs/a5d22df9eca2%2C41595%2C1733237430487.1733237452136
2024-12-03T14:50:56,309 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0/.tmp/info/1338862e3c2b418284f35dda33ca4856 is 1080, key is row1002/info:/1733237441520/Put/seqid=0
2024-12-03T14:50:56,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741845_1029 (size=9270)
2024-12-03T14:50:56,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741845_1029 (size=9270)
2024-12-03T14:50:56,314 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0/.tmp/info/1338862e3c2b418284f35dda33ca4856
2024-12-03T14:50:56,320 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0/.tmp/info/1338862e3c2b418284f35dda33ca4856 as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0/info/1338862e3c2b418284f35dda33ca4856
2024-12-03T14:50:56,326 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0/info/1338862e3c2b418284f35dda33ca4856, entries=4, sequenceid=8, filesize=9.1 K
2024-12-03T14:50:56,327 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for ab0d4a856efb6f578508d0b3f42434e0 in 41ms, sequenceid=8, compaction requested=false
2024-12-03T14:50:56,327 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ab0d4a856efb6f578508d0b3f42434e0:

2024-12-03T14:50:56,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-03T14:50:56,332 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:50:56,332 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:50:56,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:56,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:56,332 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-03T14:50:56,332 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T14:50:56,332 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1842319613, stopped=false
2024-12-03T14:50:56,332 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,33297,1733237430444
2024-12-03T14:50:56,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:50:56,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:50:56,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:56,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:50:56,334 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:50:56,334 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:50:56,334 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:50:56,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:56,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:50:56,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:50:56,334 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,41595,1733237430487' *****
2024-12-03T14:50:56,334 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:50:56,335 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(3091): Received CLOSE for ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,41595,1733237430487
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:41595.
2024-12-03T14:50:56,335 DEBUG [RS:0;a5d22df9eca2:41595 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:50:56,335 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab0d4a856efb6f578508d0b3f42434e0, disabling compactions & flushes
2024-12-03T14:50:56,335 DEBUG [RS:0;a5d22df9eca2:41595 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:50:56,335 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:50:56,335 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:56,335 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:50:56,336 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0. after waiting 0 ms
2024-12-03T14:50:56,336 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-03T14:50:56,336 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:56,337 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-03T14:50:56,337 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ab0d4a856efb6f578508d0b3f42434e0=TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.}
2024-12-03T14:50:56,337 DEBUG [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ab0d4a856efb6f578508d0b3f42434e0
2024-12-03T14:50:56,337 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:50:56,337 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:50:56,337 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:50:56,337 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:50:56,337 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:50:56,341 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/default/TestLogRolling-testLogRollOnPipelineRestart/ab0d4a856efb6f578508d0b3f42434e0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-03T14:50:56,341 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-03T14:50:56,341 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:56,341 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab0d4a856efb6f578508d0b3f42434e0:
Waiting for close lock at 1733237456335Running coprocessor pre-close hooks at 1733237456335Disabling compacts and flushes for region at 1733237456335Disabling writes for close at 1733237456336 (+1 ms)Writing region close event to WAL at 1733237456337 (+1 ms)Running coprocessor post-close hooks at 1733237456341 (+4 ms)Closed at 1733237456341
2024-12-03T14:50:56,341 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733237431425.ab0d4a856efb6f578508d0b3f42434e0.
2024-12-03T14:50:56,341 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:50:56,341 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:50:56,341 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237456337Running coprocessor pre-close hooks at 1733237456337Disabling compacts and flushes for region at 1733237456337Disabling writes for close at 1733237456337Writing region close event to WAL at 1733237456338 (+1 ms)Running coprocessor post-close hooks at 1733237456341 (+3 ms)Closed at 1733237456341
2024-12-03T14:50:56,342 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T14:50:56,369 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:56,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:56,537 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,41595,1733237430487; all regions closed.
2024-12-03T14:50:56,538 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,538 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,538 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,539 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,539 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:50:56,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741840_1023 (size=825)
2024-12-03T14:50:56,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741840_1023 (size=825)
2024-12-03T14:50:56,730 INFO  [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-03T14:50:56,730 INFO  [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-03T14:50:56,734 INFO  [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:50:56,895 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:57,173 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T14:50:57,369 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:57,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:57,896 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:58,371 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:58,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:58,898 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:59,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:59,382 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:50:59,899 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:00,176 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta after 4002ms
2024-12-03T14:51:00,177 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/WALs/a5d22df9eca2,41595,1733237430487/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta to hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/oldWALs/a5d22df9eca2%2C41595%2C1733237430487.meta.1733237431255.meta
2024-12-03T14:51:00,183 DEBUG [RS:0;a5d22df9eca2:41595 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/oldWALs
2024-12-03T14:51:00,183 INFO  [RS:0;a5d22df9eca2:41595 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C41595%2C1733237430487.meta:.meta(num 1733237456168)
2024-12-03T14:51:00,184 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,184 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,184 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,185 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,185 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741844_1028 (size=1162)
2024-12-03T14:51:00,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741844_1028 (size=1162)
2024-12-03T14:51:00,194 DEBUG [RS:0;a5d22df9eca2:41595 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/oldWALs
2024-12-03T14:51:00,194 INFO  [RS:0;a5d22df9eca2:41595 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C41595%2C1733237430487:(num 1733237456287)
2024-12-03T14:51:00,194 DEBUG [RS:0;a5d22df9eca2:41595 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:00,194 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:51:00,195 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:51:00,195 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-03T14:51:00,195 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:51:00,195 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:51:00,195 INFO  [RS:0;a5d22df9eca2:41595 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41595
2024-12-03T14:51:00,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,41595,1733237430487
2024-12-03T14:51:00,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:51:00,197 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:51:00,198 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,41595,1733237430487]
2024-12-03T14:51:00,199 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,41595,1733237430487 already deleted, retry=false
2024-12-03T14:51:00,199 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,41595,1733237430487 expired; onlineServers=0
2024-12-03T14:51:00,199 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,33297,1733237430444' *****
2024-12-03T14:51:00,199 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T14:51:00,199 INFO  [M:0;a5d22df9eca2:33297 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:51:00,199 INFO  [M:0;a5d22df9eca2:33297 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:51:00,199 DEBUG [M:0;a5d22df9eca2:33297 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T14:51:00,199 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T14:51:00,199 DEBUG [M:0;a5d22df9eca2:33297 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T14:51:00,199 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237430631 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237430631,5,FailOnTimeoutGroup]
2024-12-03T14:51:00,199 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237430633 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237430633,5,FailOnTimeoutGroup]
2024-12-03T14:51:00,199 INFO  [M:0;a5d22df9eca2:33297 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-03T14:51:00,199 INFO  [M:0;a5d22df9eca2:33297 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:51:00,199 DEBUG [M:0;a5d22df9eca2:33297 {}] master.HMaster(1795): Stopping service threads
2024-12-03T14:51:00,199 INFO  [M:0;a5d22df9eca2:33297 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T14:51:00,200 INFO  [M:0;a5d22df9eca2:33297 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:51:00,200 INFO  [M:0;a5d22df9eca2:33297 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T14:51:00,200 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T14:51:00,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T14:51:00,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:00,200 DEBUG [M:0;a5d22df9eca2:33297 {}] zookeeper.ZKUtil(347): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T14:51:00,200 WARN  [M:0;a5d22df9eca2:33297 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T14:51:00,201 INFO  [M:0;a5d22df9eca2:33297 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/.lastflushedseqids
2024-12-03T14:51:00,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741846_1030 (size=120)
2024-12-03T14:51:00,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741846_1030 (size=120)
2024-12-03T14:51:00,209 INFO  [M:0;a5d22df9eca2:33297 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-03T14:51:00,209 INFO  [M:0;a5d22df9eca2:33297 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T14:51:00,209 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:51:00,209 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:00,209 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:00,209 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:51:00,209 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:00,210 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.16 KB heapSize=29.13 KB
2024-12-03T14:51:00,210 ERROR [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData-prefix:a5d22df9eca2,33297,1733237430444 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:51:00,210 WARN  [FSHLog-0-hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData-prefix:a5d22df9eca2,33297,1733237430444 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:51:00,210 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog a5d22df9eca2%2C33297%2C1733237430444:(num 1733237430567) roll requested
2024-12-03T14:51:00,210 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C33297%2C1733237430444.1733237460210
2024-12-03T14:51:00,215 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,215 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,215 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,215 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,215 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,215 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 with entries=53, filesize=26.61 KB; new WAL /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237460210
2024-12-03T14:51:00,215 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:51:00,216 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42163,DS-5651eaf4-51ca-47eb-911c-d183237ffff9,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T14:51:00,216 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567
2024-12-03T14:51:00,216 WARN  [IPC Server handler 4 on default port 43907 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-12-03T14:51:00,216 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 after 0ms
2024-12-03T14:51:00,219 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42517:42517),(127.0.0.1/127.0.0.1:35093:35093)]
2024-12-03T14:51:00,219 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 is not closed yet, will try archiving it next time
2024-12-03T14:51:00,234 DEBUG [M:0;a5d22df9eca2:33297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/899f50f3fb5f4eca925a6705f761b2d6 is 82, key is hbase:meta,,1/info:regioninfo/1733237431280/Put/seqid=0
2024-12-03T14:51:00,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741848_1033 (size=5672)
2024-12-03T14:51:00,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741848_1033 (size=5672)
2024-12-03T14:51:00,238 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/899f50f3fb5f4eca925a6705f761b2d6
2024-12-03T14:51:00,257 DEBUG [M:0;a5d22df9eca2:33297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c1dacf31ee674295a350b4f425d82644 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733237431821/Put/seqid=0
2024-12-03T14:51:00,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741849_1034 (size=6117)
2024-12-03T14:51:00,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741849_1034 (size=6117)
2024-12-03T14:51:00,262 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.56 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c1dacf31ee674295a350b4f425d82644
2024-12-03T14:51:00,279 DEBUG [M:0;a5d22df9eca2:33297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97a0fbfdd3b5497bb3eca28924be77b9 is 69, key is a5d22df9eca2,41595,1733237430487/rs:state/1733237430717/Put/seqid=0
2024-12-03T14:51:00,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741850_1035 (size=5156)
2024-12-03T14:51:00,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741850_1035 (size=5156)
2024-12-03T14:51:00,284 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97a0fbfdd3b5497bb3eca28924be77b9
2024-12-03T14:51:00,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:00,298 INFO  [RS:0;a5d22df9eca2:41595 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:51:00,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41595-0x100a08fb5090001, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:00,298 INFO  [RS:0;a5d22df9eca2:41595 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,41595,1733237430487; zookeeper connection closed.
2024-12-03T14:51:00,298 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1938a424 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1938a424
2024-12-03T14:51:00,298 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T14:51:00,302 DEBUG [M:0;a5d22df9eca2:33297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e3ff7b0524304df799411dd9a083cfe4 is 52, key is load_balancer_on/state:d/1733237431419/Put/seqid=0
2024-12-03T14:51:00,306 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741851_1036 (size=5056)
2024-12-03T14:51:00,306 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741851_1036 (size=5056)
2024-12-03T14:51:00,306 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e3ff7b0524304df799411dd9a083cfe4
2024-12-03T14:51:00,311 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/899f50f3fb5f4eca925a6705f761b2d6 as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/899f50f3fb5f4eca925a6705f761b2d6
2024-12-03T14:51:00,316 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/899f50f3fb5f4eca925a6705f761b2d6, entries=8, sequenceid=56, filesize=5.5 K
2024-12-03T14:51:00,317 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c1dacf31ee674295a350b4f425d82644 as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c1dacf31ee674295a350b4f425d82644
2024-12-03T14:51:00,324 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c1dacf31ee674295a350b4f425d82644, entries=6, sequenceid=56, filesize=6.0 K
2024-12-03T14:51:00,325 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97a0fbfdd3b5497bb3eca28924be77b9 as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97a0fbfdd3b5497bb3eca28924be77b9
2024-12-03T14:51:00,330 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97a0fbfdd3b5497bb3eca28924be77b9, entries=1, sequenceid=56, filesize=5.0 K
2024-12-03T14:51:00,331 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e3ff7b0524304df799411dd9a083cfe4 as hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e3ff7b0524304df799411dd9a083cfe4
2024-12-03T14:51:00,335 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e3ff7b0524304df799411dd9a083cfe4, entries=1, sequenceid=56, filesize=4.9 K
2024-12-03T14:51:00,336 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=56, compaction requested=false
2024-12-03T14:51:00,338 INFO  [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:00,338 DEBUG [M:0;a5d22df9eca2:33297 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237460209Disabling compacts and flushes for region at 1733237460209Disabling writes for close at 1733237460209Obtaining lock to block concurrent updates at 1733237460210 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237460210Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23714, getHeapSize=29768, getOffHeapSize=0, getCellsCount=67 at 1733237460210Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237460220 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237460220Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237460233 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237460233Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237460243 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237460256 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237460257 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237460266 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237460279 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237460279Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733237460289 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733237460301 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733237460301Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27df0543: reopening flushed file at 1733237460310 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e17d19f: reopening flushed file at 1733237460316 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fe00b90: reopening flushed file at 1733237460324 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52d1a86b: reopening flushed file at 1733237460330 (+6 ms)Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=56, compaction requested=false at 1733237460336 (+6 ms)Writing region close event to WAL at 1733237460338 (+2 ms)Closed at 1733237460338
2024-12-03T14:51:00,338 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,338 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,338 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,338 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,338 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:00,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741847_1031 (size=757)
2024-12-03T14:51:00,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33337 is added to blk_1073741847_1031 (size=757)
2024-12-03T14:51:00,373 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:00,382 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:00,423 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T14:51:00,900 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:01,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,343 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,365 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,365 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,365 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,373 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:01,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:01,873 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:51:01,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,878 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:01,901 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:01,904 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:02,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:51:02,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-03T14:51:02,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-03T14:51:02,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T14:51:02,374 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:02,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:02,902 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:03,174 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T14:51:03,375 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:03,384 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:03,903 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:04,217 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 after 4001ms
2024-12-03T14:51:04,218 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/WALs/a5d22df9eca2,33297,1733237430444/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 to hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/oldWALs/a5d22df9eca2%2C33297%2C1733237430444.1733237430567
2024-12-03T14:51:04,226 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/MasterData/oldWALs/a5d22df9eca2%2C33297%2C1733237430444.1733237430567 to hdfs://localhost:43907/user/jenkins/test-data/205b00ce-3c61-815a-b2d6-e080c53590b3/oldWALs/a5d22df9eca2%2C33297%2C1733237430444.1733237430567$masterlocalwal$
2024-12-03T14:51:04,226 INFO  [M:0;a5d22df9eca2:33297 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-03T14:51:04,226 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:51:04,226 INFO  [M:0;a5d22df9eca2:33297 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33297
2024-12-03T14:51:04,226 INFO  [M:0;a5d22df9eca2:33297 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:51:04,328 INFO  [M:0;a5d22df9eca2:33297 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:51:04,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:04,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33297-0x100a08fb5090000, quorum=127.0.0.1:53587, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:04,333 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@528eeea6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:04,333 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@20466b6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:51:04,333 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:51:04,334 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2026736d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:51:04,334 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@760c54cf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,STOPPED}
2024-12-03T14:51:04,335 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:51:04,335 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-594567723-172.17.0.2-1733237429890 (Datanode Uuid 31376101-3aac-42b1-afb1-79f3c7db9fdd) service to localhost/127.0.0.1:43907
2024-12-03T14:51:04,336 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data3/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:04,336 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data4/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:04,336 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:51:04,337 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:51:04,337 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:51:04,340 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ca8564b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:04,340 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@fa662a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:51:04,340 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:51:04,340 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@167fd01b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:51:04,340 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@718ea2f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,STOPPED}
2024-12-03T14:51:04,342 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:51:04,342 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:51:04,342 WARN  [BP-594567723-172.17.0.2-1733237429890 heartbeating to localhost/127.0.0.1:43907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-594567723-172.17.0.2-1733237429890 (Datanode Uuid 7dbe9ada-1e0f-4f81-8c47-1dc93f83a813) service to localhost/127.0.0.1:43907
2024-12-03T14:51:04,342 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:51:04,343 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data1/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:04,343 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/cluster_916fb2e7-8210-c4ac-57a9-074575b85295/data/data2/current/BP-594567723-172.17.0.2-1733237429890 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:04,343 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:51:04,349 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1915705e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:51:04,350 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17b2a9ba{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:51:04,350 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:51:04,350 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27a49013{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:51:04,350 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7bd03e52{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir/,STOPPED}
2024-12-03T14:51:04,356 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-03T14:51:04,376 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:04,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-03T14:51:04,383 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=180 (was 155)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:43907
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43907
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43907
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43907
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43907 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43907
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43907 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43907 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 442) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=99 (was 99), ProcessCount=11 (was 11), AvailableMemoryMB=4654 (was 4899)
2024-12-03T14:51:04,385 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:04,389 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=180, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=99, ProcessCount=11, AvailableMemoryMB=4654
2024-12-03T14:51:04,389 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T14:51:04,389 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.log.dir so I do NOT create it in target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae
2024-12-03T14:51:04,389 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d92391e2-28a5-01e6-f101-830c1d10544d/hadoop.tmp.dir so I do NOT create it in target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae
2024-12-03T14:51:04,389 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c, deleteOnExit=true
2024-12-03T14:51:04,389 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/test.cache.data in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-03T14:51:04,390 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T14:51:04,390 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:51:04,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:51:04,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T14:51:04,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/nfs.dump.dir in system properties and HBase conf
2024-12-03T14:51:04,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/java.io.tmpdir in system properties and HBase conf
2024-12-03T14:51:04,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:51:04,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T14:51:04,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T14:51:04,403 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:51:04,444 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:04,448 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:51:04,449 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:51:04,449 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:51:04,449 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:51:04,449 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:04,450 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78b6351{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:51:04,450 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5bbcc3bf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:51:04,539 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@232381c8{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/java.io.tmpdir/jetty-localhost-42021-hadoop-hdfs-3_4_1-tests_jar-_-any-14437524069458020652/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:51:04,540 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@290f188f{HTTP/1.1, (http/1.1)}{localhost:42021}
2024-12-03T14:51:04,540 INFO  [Time-limited test {}] server.Server(415): Started @187847ms
2024-12-03T14:51:04,550 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:51:04,590 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:04,592 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:51:04,593 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:51:04,593 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:51:04,593 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:51:04,593 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b3167c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:51:04,593 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fa328f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:51:04,683 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d027a3b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/java.io.tmpdir/jetty-localhost-39223-hadoop-hdfs-3_4_1-tests_jar-_-any-1318001423475809069/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:04,684 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b5b6aa1{HTTP/1.1, (http/1.1)}{localhost:39223}
2024-12-03T14:51:04,684 INFO  [Time-limited test {}] server.Server(415): Started @187991ms
2024-12-03T14:51:04,685 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:51:04,708 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:04,710 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:51:04,711 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:51:04,711 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:51:04,711 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:51:04,711 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b24fbcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:51:04,711 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16aeea80{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:51:04,736 WARN  [Thread-1640 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data2/current/BP-1518659009-172.17.0.2-1733237464412/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:04,736 WARN  [Thread-1639 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data1/current/BP-1518659009-172.17.0.2-1733237464412/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:04,749 WARN  [Thread-1618 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:51:04,751 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaea6f7742862c82f with lease ID 0xc9b5ae4adf7e5706: Processing first storage report for DS-abd28e70-2c30-45bf-a2dd-ccd4a82f26a8 from datanode DatanodeRegistration(127.0.0.1:33879, datanodeUuid=7badb565-01a6-4ee7-b8fb-7e1b2bd45105, infoPort=39623, infoSecurePort=0, ipcPort=36097, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412)
2024-12-03T14:51:04,751 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaea6f7742862c82f with lease ID 0xc9b5ae4adf7e5706: from storage DS-abd28e70-2c30-45bf-a2dd-ccd4a82f26a8 node DatanodeRegistration(127.0.0.1:33879, datanodeUuid=7badb565-01a6-4ee7-b8fb-7e1b2bd45105, infoPort=39623, infoSecurePort=0, ipcPort=36097, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:04,751 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaea6f7742862c82f with lease ID 0xc9b5ae4adf7e5706: Processing first storage report for DS-876baaa1-236e-4a7b-85d8-7e9cf01797f7 from datanode DatanodeRegistration(127.0.0.1:33879, datanodeUuid=7badb565-01a6-4ee7-b8fb-7e1b2bd45105, infoPort=39623, infoSecurePort=0, ipcPort=36097, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412)
2024-12-03T14:51:04,751 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaea6f7742862c82f with lease ID 0xc9b5ae4adf7e5706: from storage DS-876baaa1-236e-4a7b-85d8-7e9cf01797f7 node DatanodeRegistration(127.0.0.1:33879, datanodeUuid=7badb565-01a6-4ee7-b8fb-7e1b2bd45105, infoPort=39623, infoSecurePort=0, ipcPort=36097, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:04,804 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d8d7f9b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/java.io.tmpdir/jetty-localhost-45833-hadoop-hdfs-3_4_1-tests_jar-_-any-5647600140466677458/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:04,804 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@190ad9e7{HTTP/1.1, (http/1.1)}{localhost:45833}
2024-12-03T14:51:04,804 INFO  [Time-limited test {}] server.Server(415): Started @188111ms
2024-12-03T14:51:04,805 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:51:04,857 WARN  [Thread-1665 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data3/current/BP-1518659009-172.17.0.2-1733237464412/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:04,857 WARN  [Thread-1666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data4/current/BP-1518659009-172.17.0.2-1733237464412/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:04,875 WARN  [Thread-1654 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:51:04,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x819fad68a6374d07 with lease ID 0xc9b5ae4adf7e5707: Processing first storage report for DS-10d86858-aa92-4ed9-b0bf-99d7b46b6081 from datanode DatanodeRegistration(127.0.0.1:37447, datanodeUuid=3fb0bf93-3031-4140-a3e0-ed7749156b60, infoPort=37949, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412)
2024-12-03T14:51:04,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x819fad68a6374d07 with lease ID 0xc9b5ae4adf7e5707: from storage DS-10d86858-aa92-4ed9-b0bf-99d7b46b6081 node DatanodeRegistration(127.0.0.1:37447, datanodeUuid=3fb0bf93-3031-4140-a3e0-ed7749156b60, infoPort=37949, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:04,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x819fad68a6374d07 with lease ID 0xc9b5ae4adf7e5707: Processing first storage report for DS-27a1f908-7a54-4818-b4e9-648cc48c9031 from datanode DatanodeRegistration(127.0.0.1:37447, datanodeUuid=3fb0bf93-3031-4140-a3e0-ed7749156b60, infoPort=37949, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412)
2024-12-03T14:51:04,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x819fad68a6374d07 with lease ID 0xc9b5ae4adf7e5707: from storage DS-27a1f908-7a54-4818-b4e9-648cc48c9031 node DatanodeRegistration(127.0.0.1:37447, datanodeUuid=3fb0bf93-3031-4140-a3e0-ed7749156b60, infoPort=37949, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=669673311;c=1733237464412), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:04,904 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:04,936 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae
2024-12-03T14:51:04,939 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/zookeeper_0, clientPort=54868, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T14:51:04,940 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54868
2024-12-03T14:51:04,940 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:04,941 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:04,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:51:04,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:51:04,950 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23 with version=8
2024-12-03T14:51:04,950 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase-staging
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-03T14:51:04,952 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:51:04,953 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44923
2024-12-03T14:51:04,954 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44923 connecting to ZooKeeper ensemble=127.0.0.1:54868
2024-12-03T14:51:04,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449230x0, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:51:04,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44923-0x100a0903bd90000 connected
2024-12-03T14:51:04,969 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:04,970 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:04,972 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:51:04,972 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23, hbase.cluster.distributed=false
2024-12-03T14:51:04,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:51:04,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44923
2024-12-03T14:51:04,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44923
2024-12-03T14:51:04,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44923
2024-12-03T14:51:04,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44923
2024-12-03T14:51:04,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44923
2024-12-03T14:51:04,993 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:51:04,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:04,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:04,993 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:51:04,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:04,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:51:04,993 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:51:04,994 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:51:04,994 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42723
2024-12-03T14:51:04,995 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42723 connecting to ZooKeeper ensemble=127.0.0.1:54868
2024-12-03T14:51:04,996 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:04,997 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:05,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427230x0, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:51:05,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427230x0, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:51:05,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42723-0x100a0903bd90001 connected
2024-12-03T14:51:05,001 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:51:05,001 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:51:05,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T14:51:05,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:51:05,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42723
2024-12-03T14:51:05,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42723
2024-12-03T14:51:05,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42723
2024-12-03T14:51:05,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42723
2024-12-03T14:51:05,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42723
2024-12-03T14:51:05,013 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:44923
2024-12-03T14:51:05,014 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:05,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:05,015 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T14:51:05,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,016 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:51:05,017 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,44923,1733237464952 from backup master directory
2024-12-03T14:51:05,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:05,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:05,017 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:51:05,017 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,021 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/hbase.id] with ID: 8d48cf2a-b46b-46d8-a8ad-62f08df0a04a
2024-12-03T14:51:05,021 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/.tmp/hbase.id
2024-12-03T14:51:05,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:51:05,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:51:05,027 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/.tmp/hbase.id]:[hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/hbase.id]
2024-12-03T14:51:05,038 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:05,038 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-03T14:51:05,039 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-03T14:51:05,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:51:05,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:51:05,049 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:51:05,050 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T14:51:05,050 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:51:05,060 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:51:05,060 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:51:05,061 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store
2024-12-03T14:51:05,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:51:05,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:51:05,074 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:05,075 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:51:05,075 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:05,075 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:05,075 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:51:05,075 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:05,075 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:05,075 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237465074Disabling compacts and flushes for region at 1733237465074Disabling writes for close at 1733237465075 (+1 ms)Writing region close event to WAL at 1733237465075Closed at 1733237465075
2024-12-03T14:51:05,075 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/.initializing
2024-12-03T14:51:05,075 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/WALs/a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,078 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C44923%2C1733237464952, suffix=, logDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/WALs/a5d22df9eca2,44923,1733237464952, archiveDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/oldWALs, maxLogs=10
2024-12-03T14:51:05,078 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C44923%2C1733237464952.1733237465078
2024-12-03T14:51:05,084 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/WALs/a5d22df9eca2,44923,1733237464952/a5d22df9eca2%2C44923%2C1733237464952.1733237465078
2024-12-03T14:51:05,084 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39623:39623),(127.0.0.1/127.0.0.1:37949:37949)]
2024-12-03T14:51:05,086 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:51:05,086 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:05,086 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,086 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,087 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:05,088 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T14:51:05,088 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,089 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,089 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:05,090 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T14:51:05,090 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,090 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:05,091 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:05,092 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T14:51:05,092 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,092 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:05,092 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:05,093 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T14:51:05,093 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,093 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:05,094 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,094 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,095 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,096 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,096 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,097 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T14:51:05,098 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:05,101 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:51:05,101 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=846409, jitterRate=0.07626481354236603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T14:51:05,102 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733237465086Initializing all the Stores at 1733237465087 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237465087Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237465087Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237465087Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237465087Cleaning up temporary data from old regions at 1733237465096 (+9 ms)Region opened successfully at 1733237465102 (+6 ms)
2024-12-03T14:51:05,102 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T14:51:05,105 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340635cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:51:05,106 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-03T14:51:05,106 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T14:51:05,106 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T14:51:05,107 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T14:51:05,107 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T14:51:05,108 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-03T14:51:05,108 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T14:51:05,110 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T14:51:05,111 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T14:51:05,112 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-03T14:51:05,112 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T14:51:05,113 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T14:51:05,113 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-03T14:51:05,114 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T14:51:05,114 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T14:51:05,115 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-03T14:51:05,116 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T14:51:05,116 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T14:51:05,118 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T14:51:05,118 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T14:51:05,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:51:05,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:51:05,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,120 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,44923,1733237464952, sessionid=0x100a0903bd90000, setting cluster-up flag (Was=false)
2024-12-03T14:51:05,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,124 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T14:51:05,125 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,130 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T14:51:05,131 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,132 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-03T14:51:05,134 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:05,134 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-03T14:51:05,135 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T14:51:05,135 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,44923,1733237464952
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T14:51:05,136 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:05,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:05,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:05,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:05,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10
2024-12-03T14:51:05,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:51:05,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,139 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237495139
2024-12-03T14:51:05,139 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T14:51:05,139 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T14:51:05,139 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T14:51:05,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T14:51:05,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T14:51:05,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T14:51:05,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,140 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:05,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T14:51:05,140 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-03T14:51:05,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T14:51:05,140 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T14:51:05,141 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T14:51:05,141 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T14:51:05,141 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237465141,5,FailOnTimeoutGroup]
2024-12-03T14:51:05,141 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,142 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T14:51:05,142 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237465141,5,FailOnTimeoutGroup]
2024-12-03T14:51:05,142 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,142 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T14:51:05,142 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,142 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:51:05,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:51:05,150 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-03T14:51:05,150 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23
2024-12-03T14:51:05,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:51:05,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:51:05,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:05,161 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:51:05,162 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:51:05,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,163 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,163 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:51:05,164 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:51:05,164 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,164 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,164 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:51:05,165 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:51:05,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,166 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,166 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:51:05,167 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:51:05,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,167 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:51:05,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740
2024-12-03T14:51:05,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740
2024-12-03T14:51:05,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:51:05,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:51:05,170 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:51:05,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:51:05,173 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:51:05,173 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=722466, jitterRate=-0.08133751153945923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:51:05,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733237465159Initializing all the Stores at 1733237465160 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237465160Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237465161 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237465161Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237465161Cleaning up temporary data from old regions at 1733237465170 (+9 ms)Region opened successfully at 1733237465173 (+3 ms)
2024-12-03T14:51:05,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:51:05,174 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:51:05,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:51:05,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:51:05,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:51:05,174 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:51:05,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237465174Disabling compacts and flushes for region at 1733237465174Disabling writes for close at 1733237465174Writing region close event to WAL at 1733237465174Closed at 1733237465174
2024-12-03T14:51:05,175 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:05,175 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-03T14:51:05,175 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T14:51:05,177 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:51:05,178 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T14:51:05,206 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(746): ClusterId : 8d48cf2a-b46b-46d8-a8ad-62f08df0a04a
2024-12-03T14:51:05,206 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:51:05,208 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:51:05,208 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:51:05,210 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:51:05,210 DEBUG [RS:0;a5d22df9eca2:42723 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33841934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:51:05,220 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:42723
2024-12-03T14:51:05,220 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:51:05,220 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:51:05,220 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:51:05,221 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,44923,1733237464952 with port=42723, startcode=1733237464993
2024-12-03T14:51:05,221 DEBUG [RS:0;a5d22df9eca2:42723 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:51:05,222 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51557, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:51:05,223 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44923 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,223 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44923 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,224 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23
2024-12-03T14:51:05,224 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46013
2024-12-03T14:51:05,224 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:51:05,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:51:05,226 DEBUG [RS:0;a5d22df9eca2:42723 {}] zookeeper.ZKUtil(111): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,226 WARN  [RS:0;a5d22df9eca2:42723 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:51:05,226 INFO  [RS:0;a5d22df9eca2:42723 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:51:05,226 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,227 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,42723,1733237464993]
2024-12-03T14:51:05,229 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:51:05,230 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:51:05,231 INFO  [RS:0;a5d22df9eca2:42723 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:51:05,231 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,231 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:51:05,232 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:51:05,232 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,232 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:05,233 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:51:05,233 DEBUG [RS:0;a5d22df9eca2:42723 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:51:05,233 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,233 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,233 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,233 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,233 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,233 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42723,1733237464993-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:51:05,247 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:51:05,247 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,42723,1733237464993-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,247 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,247 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.Replication(171): a5d22df9eca2,42723,1733237464993 started
2024-12-03T14:51:05,259 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,259 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,42723,1733237464993, RpcServer on a5d22df9eca2/172.17.0.2:42723, sessionid=0x100a0903bd90001
2024-12-03T14:51:05,259 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:51:05,259 DEBUG [RS:0;a5d22df9eca2:42723 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,259 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,42723,1733237464993'
2024-12-03T14:51:05,259 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:51:05,260 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:51:05,260 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:51:05,260 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:51:05,260 DEBUG [RS:0;a5d22df9eca2:42723 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,260 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,42723,1733237464993'
2024-12-03T14:51:05,260 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:51:05,261 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:51:05,261 DEBUG [RS:0;a5d22df9eca2:42723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:51:05,261 INFO  [RS:0;a5d22df9eca2:42723 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:51:05,261 INFO  [RS:0;a5d22df9eca2:42723 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:51:05,328 WARN  [a5d22df9eca2:44923 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-03T14:51:05,364 INFO  [RS:0;a5d22df9eca2:42723 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C42723%2C1733237464993, suffix=, logDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993, archiveDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/oldWALs, maxLogs=32
2024-12-03T14:51:05,365 INFO  [RS:0;a5d22df9eca2:42723 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C42723%2C1733237464993.1733237465365
2024-12-03T14:51:05,375 INFO  [RS:0;a5d22df9eca2:42723 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237465365
2024-12-03T14:51:05,377 DEBUG [RS:0;a5d22df9eca2:42723 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37949:37949),(127.0.0.1/127.0.0.1:39623:39623)]
2024-12-03T14:51:05,377 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:05,386 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:05,578 DEBUG [a5d22df9eca2:44923 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T14:51:05,579 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,583 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,42723,1733237464993, state=OPENING
2024-12-03T14:51:05,585 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T14:51:05,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:05,588 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:51:05,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:05,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:05,588 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,42723,1733237464993}]
2024-12-03T14:51:05,743 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T14:51:05,747 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52121, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T14:51:05,754 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-03T14:51:05,754 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:51:05,757 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C42723%2C1733237464993.meta, suffix=.meta, logDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993, archiveDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/oldWALs, maxLogs=32
2024-12-03T14:51:05,758 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C42723%2C1733237464993.meta.1733237465758.meta
2024-12-03T14:51:05,765 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.meta.1733237465758.meta
2024-12-03T14:51:05,767 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39623:39623),(127.0.0.1/127.0.0.1:37949:37949)]
2024-12-03T14:51:05,768 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:51:05,769 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T14:51:05,769 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T14:51:05,769 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T14:51:05,769 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T14:51:05,769 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:05,769 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-03T14:51:05,769 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-03T14:51:05,772 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:51:05,773 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:51:05,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,774 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,774 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:51:05,775 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:51:05,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,776 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,776 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:51:05,777 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:51:05,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,777 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,777 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:51:05,778 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:51:05,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,779 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:05,779 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:51:05,780 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740
2024-12-03T14:51:05,781 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740
2024-12-03T14:51:05,783 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:51:05,783 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:51:05,784 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:51:05,785 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:51:05,786 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=729516, jitterRate=-0.07237254083156586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:51:05,786 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-03T14:51:05,787 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733237465770Writing region info on filesystem at 1733237465770Initializing all the Stores at 1733237465771 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237465771Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237465771Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237465771Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237465771Cleaning up temporary data from old regions at 1733237465783 (+12 ms)Running coprocessor post-open hooks at 1733237465786 (+3 ms)Region opened successfully at 1733237465787 (+1 ms)
2024-12-03T14:51:05,788 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237465743
2024-12-03T14:51:05,790 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T14:51:05,790 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-03T14:51:05,791 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,792 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,42723,1733237464993, state=OPEN
2024-12-03T14:51:05,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:51:05,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:51:05,794 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:05,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:05,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:05,797 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T14:51:05,797 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,42723,1733237464993 in 206 msec
2024-12-03T14:51:05,799 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T14:51:05,799 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 622 msec
2024-12-03T14:51:05,800 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:05,800 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-03T14:51:05,801 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:51:05,801 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,42723,1733237464993, seqNum=-1]
2024-12-03T14:51:05,801 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:51:05,802 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43457, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:51:05,808 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 673 msec
2024-12-03T14:51:05,808 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237465808, completionTime=-1
2024-12-03T14:51:05,808 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T14:51:05,808 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-03T14:51:05,810 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-03T14:51:05,810 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237525810
2024-12-03T14:51:05,810 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237585810
2024-12-03T14:51:05,810 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-03T14:51:05,811 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44923,1733237464952-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,811 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44923,1733237464952-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,811 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44923,1733237464952-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,811 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:44923, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,811 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,811 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,813 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-03T14:51:05,814 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.797sec
2024-12-03T14:51:05,814 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T14:51:05,814 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T14:51:05,815 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T14:51:05,815 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T14:51:05,815 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T14:51:05,815 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44923,1733237464952-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:51:05,815 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44923,1733237464952-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T14:51:05,817 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-03T14:51:05,817 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T14:51:05,817 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44923,1733237464952-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:05,905 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:05,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f17b515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:51:05,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,44923,-1 for getting cluster id
2024-12-03T14:51:05,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-03T14:51:05,908 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8d48cf2a-b46b-46d8-a8ad-62f08df0a04a'
2024-12-03T14:51:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-03T14:51:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8d48cf2a-b46b-46d8-a8ad-62f08df0a04a"

2024-12-03T14:51:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac51ce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:51:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,44923,-1]
2024-12-03T14:51:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-03T14:51:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:05,910 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-03T14:51:05,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c2896a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:51:05,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:51:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,42723,1733237464993, seqNum=-1]
2024-12-03T14:51:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:51:05,914 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:51:05,916 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,917 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:05,919 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-03T14:51:05,919 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-03T14:51:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f022144
2024-12-03T14:51:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T14:51:05,921 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T14:51:05,922 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T14:51:05,922 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T14:51:05,922 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:51:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:05,925 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T14:51:05,925 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:05,925 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-03T14:51:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:51:05,926 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T14:51:05,933 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741835_1011 (size=405)
2024-12-03T14:51:05,933 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741835_1011 (size=405)
2024-12-03T14:51:05,935 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cc24e7f75d95dc52a9c99aae568056b2, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23
2024-12-03T14:51:05,944 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741836_1012 (size=88)
2024-12-03T14:51:05,944 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741836_1012 (size=88)
2024-12-03T14:51:06,345 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:06,345 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing cc24e7f75d95dc52a9c99aae568056b2, disabling compactions & flushes
2024-12-03T14:51:06,345 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:06,345 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:06,345 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2. after waiting 0 ms
2024-12-03T14:51:06,345 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:06,345 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:06,346 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for cc24e7f75d95dc52a9c99aae568056b2:
Waiting for close lock at 1733237466345Disabling compacts and flushes for region at 1733237466345Disabling writes for close at 1733237466345Writing region close event to WAL at 1733237466345Closed at 1733237466345
2024-12-03T14:51:06,348 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T14:51:06,348 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733237466348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237466348"}]},"ts":"1733237466348"}
2024-12-03T14:51:06,352 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-03T14:51:06,353 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T14:51:06,353 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237466353"}]},"ts":"1733237466353"}
2024-12-03T14:51:06,356 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-03T14:51:06,356 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=cc24e7f75d95dc52a9c99aae568056b2, ASSIGN}]
2024-12-03T14:51:06,358 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=cc24e7f75d95dc52a9c99aae568056b2, ASSIGN
2024-12-03T14:51:06,359 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=cc24e7f75d95dc52a9c99aae568056b2, ASSIGN; state=OFFLINE, location=a5d22df9eca2,42723,1733237464993; forceNewPlan=false, retain=false
2024-12-03T14:51:06,377 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:06,386 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:06,510 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc24e7f75d95dc52a9c99aae568056b2, regionState=OPENING, regionLocation=a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:06,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=cc24e7f75d95dc52a9c99aae568056b2, ASSIGN because future has completed
2024-12-03T14:51:06,513 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc24e7f75d95dc52a9c99aae568056b2, server=a5d22df9eca2,42723,1733237464993}]
2024-12-03T14:51:06,672 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:06,672 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cc24e7f75d95dc52a9c99aae568056b2, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:51:06,673 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,673 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:06,673 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,673 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,676 INFO  [StoreOpener-cc24e7f75d95dc52a9c99aae568056b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cc24e7f75d95dc52a9c99aae568056b2 
2024-12-03T14:51:06,679 INFO  [StoreOpener-cc24e7f75d95dc52a9c99aae568056b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc24e7f75d95dc52a9c99aae568056b2 columnFamilyName info
2024-12-03T14:51:06,679 DEBUG [StoreOpener-cc24e7f75d95dc52a9c99aae568056b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:06,679 INFO  [StoreOpener-cc24e7f75d95dc52a9c99aae568056b2-1 {}] regionserver.HStore(327): Store=cc24e7f75d95dc52a9c99aae568056b2/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:06,680 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,681 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,681 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,682 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,682 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,684 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,688 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:51:06,689 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cc24e7f75d95dc52a9c99aae568056b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811168, jitterRate=0.03145468235015869}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T14:51:06,689 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:06,690 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cc24e7f75d95dc52a9c99aae568056b2:
Running coprocessor pre-open hook at 1733237466673Writing region info on filesystem at 1733237466673Initializing all the Stores at 1733237466675 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237466675Cleaning up temporary data from old regions at 1733237466682 (+7 ms)Running coprocessor post-open hooks at 1733237466689 (+7 ms)Region opened successfully at 1733237466690 (+1 ms)
2024-12-03T14:51:06,691 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2., pid=6, masterSystemTime=1733237466667
2024-12-03T14:51:06,694 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:06,694 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:06,695 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc24e7f75d95dc52a9c99aae568056b2, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:06,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc24e7f75d95dc52a9c99aae568056b2, server=a5d22df9eca2,42723,1733237464993 because future has completed
2024-12-03T14:51:06,702 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T14:51:06,702 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cc24e7f75d95dc52a9c99aae568056b2, server=a5d22df9eca2,42723,1733237464993 in 186 msec
2024-12-03T14:51:06,706 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T14:51:06,706 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=cc24e7f75d95dc52a9c99aae568056b2, ASSIGN in 346 msec
2024-12-03T14:51:06,708 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T14:51:06,708 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237466708"}]},"ts":"1733237466708"}
2024-12-03T14:51:06,711 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-03T14:51:06,712 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T14:51:06,714 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 790 msec
2024-12-03T14:51:06,906 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:07,378 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:07,387 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:07,872 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:51:07,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,893 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,893 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,893 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,893 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,893 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:51:07,907 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:08,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:08,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:08,908 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:09,380 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:09,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:09,908 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:10,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:10,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:10,910 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:11,229 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T14:51:11,230 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-03T14:51:11,382 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:11,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:11,911 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:12,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-03T14:51:12,366 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-03T14:51:12,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:51:12,367 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-03T14:51:12,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-03T14:51:12,367 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-03T14:51:12,368 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:12,368 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-03T14:51:12,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:12,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:12,912 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:13,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:13,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:13,913 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:14,385 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:14,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:14,914 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:15,386 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:15,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:15,915 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:51:16,014 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-03T14:51:16,014 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-03T14:51:16,019 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:16,019 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2., hostname=a5d22df9eca2,42723,1733237464993, seqNum=2]
2024-12-03T14:51:16,031 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:16,038 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T14:51:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-03T14:51:16,039 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T14:51:16,041 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T14:51:16,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-03T14:51:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:16,209 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing cc24e7f75d95dc52a9c99aae568056b2 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T14:51:16,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/8e6e33a3285e46e985411d413a22ddd1 is 1080, key is row0001/info:/1733237476025/Put/seqid=0
2024-12-03T14:51:16,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741837_1013 (size=6033)
2024-12-03T14:51:16,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741837_1013 (size=6033)
2024-12-03T14:51:16,233 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/8e6e33a3285e46e985411d413a22ddd1
2024-12-03T14:51:16,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/8e6e33a3285e46e985411d413a22ddd1 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/8e6e33a3285e46e985411d413a22ddd1
2024-12-03T14:51:16,244 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/8e6e33a3285e46e985411d413a22ddd1, entries=1, sequenceid=5, filesize=5.9 K
2024-12-03T14:51:16,245 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for cc24e7f75d95dc52a9c99aae568056b2 in 36ms, sequenceid=5, compaction requested=false
2024-12-03T14:51:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for cc24e7f75d95dc52a9c99aae568056b2:

2024-12-03T14:51:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:16,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-03T14:51:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-03T14:51:16,253 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-03T14:51:16,253 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 209 msec
2024-12-03T14:51:16,256 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 222 msec
2024-12-03T14:51:16,387 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:16,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:16,916 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:17,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:17,395 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:17,917 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:18,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:18,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:18,918 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:19,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:19,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:19,919 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:19,920 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716 after 68076ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:51:20,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:20,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:20,922 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:21,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:21,399 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:21,923 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:22,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:22,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:22,924 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:23,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:23,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:23,924 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:24,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:24,403 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:24,925 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:25,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:25,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:25,926 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-03T14:51:26,075 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-03T14:51:26,081 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-03T14:51:26,085 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T14:51:26,088 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T14:51:26,088 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T14:51:26,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-03T14:51:26,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:26,244 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing cc24e7f75d95dc52a9c99aae568056b2 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T14:51:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/77447121df43463ba451540935f25ce1 is 1080, key is row0002/info:/1733237486077/Put/seqid=0
2024-12-03T14:51:26,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741838_1014 (size=6033)
2024-12-03T14:51:26,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741838_1014 (size=6033)
2024-12-03T14:51:26,262 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/77447121df43463ba451540935f25ce1
2024-12-03T14:51:26,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/77447121df43463ba451540935f25ce1 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/77447121df43463ba451540935f25ce1
2024-12-03T14:51:26,272 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/77447121df43463ba451540935f25ce1, entries=1, sequenceid=9, filesize=5.9 K
2024-12-03T14:51:26,273 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for cc24e7f75d95dc52a9c99aae568056b2 in 29ms, sequenceid=9, compaction requested=false
2024-12-03T14:51:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for cc24e7f75d95dc52a9c99aae568056b2:

2024-12-03T14:51:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-03T14:51:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-03T14:51:26,277 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-03T14:51:26,277 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-12-03T14:51:26,279 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 196 msec
2024-12-03T14:51:26,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:26,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:26,927 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:27,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:27,401 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649 after 68074ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:51:27,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:27,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta after 68063ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T14:51:27,928 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:28,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:28,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:28,929 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:29,403 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:29,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:29,930 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:30,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:30,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:30,931 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:31,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:31,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:31,932 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:32,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:32,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:32,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:33,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:33,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:33,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:34,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:34,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:34,935 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:34,935 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T14:51:35,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:35,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:35,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-03T14:51:36,153 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-03T14:51:36,157 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C42723%2C1733237464993.1733237496156
2024-12-03T14:51:36,163 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:36,163 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:36,164 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:36,164 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:36,164 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:36,164 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237465365 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237496156
2024-12-03T14:51:36,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741833_1009 (size=5546)
2024-12-03T14:51:36,167 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741833_1009 (size=5546)
2024-12-03T14:51:36,170 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39623:39623),(127.0.0.1/127.0.0.1:37949:37949)]
2024-12-03T14:51:36,171 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-03T14:51:36,173 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T14:51:36,174 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T14:51:36,174 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T14:51:36,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-03T14:51:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:36,328 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing cc24e7f75d95dc52a9c99aae568056b2 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T14:51:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/21d594b14b8f4fd6848cd32f355d2544 is 1080, key is row0003/info:/1733237496154/Put/seqid=0
2024-12-03T14:51:36,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741840_1016 (size=6033)
2024-12-03T14:51:36,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741840_1016 (size=6033)
2024-12-03T14:51:36,341 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/21d594b14b8f4fd6848cd32f355d2544
2024-12-03T14:51:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/21d594b14b8f4fd6848cd32f355d2544 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/21d594b14b8f4fd6848cd32f355d2544
2024-12-03T14:51:36,359 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/21d594b14b8f4fd6848cd32f355d2544, entries=1, sequenceid=13, filesize=5.9 K
2024-12-03T14:51:36,360 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for cc24e7f75d95dc52a9c99aae568056b2 in 32ms, sequenceid=13, compaction requested=true
2024-12-03T14:51:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for cc24e7f75d95dc52a9c99aae568056b2:

2024-12-03T14:51:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-03T14:51:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-03T14:51:36,364 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-03T14:51:36,364 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 188 msec
2024-12-03T14:51:36,366 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 194 msec
2024-12-03T14:51:36,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:36,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:36,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:37,411 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:37,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:37,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:38,411 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:38,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:38,938 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:39,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:39,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:39,939 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:40,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:40,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:40,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:41,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:41,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:41,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:42,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:42,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:42,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:43,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:43,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:43,942 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:44,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:44,419 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:44,942 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:45,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:45,419 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:45,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-03T14:51:46,183 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-03T14:51:46,184 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:51:46,185 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:51:46,185 DEBUG [Time-limited test {}] regionserver.HStore(1541): cc24e7f75d95dc52a9c99aae568056b2/info is initiating minor compaction (all files)
2024-12-03T14:51:46,185 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:51:46,186 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:46,186 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of cc24e7f75d95dc52a9c99aae568056b2/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:46,186 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/8e6e33a3285e46e985411d413a22ddd1, hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/77447121df43463ba451540935f25ce1, hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/21d594b14b8f4fd6848cd32f355d2544] into tmpdir=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp, totalSize=17.7 K
2024-12-03T14:51:46,186 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8e6e33a3285e46e985411d413a22ddd1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733237476025
2024-12-03T14:51:46,187 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 77447121df43463ba451540935f25ce1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733237486077
2024-12-03T14:51:46,187 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 21d594b14b8f4fd6848cd32f355d2544, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733237496154
2024-12-03T14:51:46,199 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): cc24e7f75d95dc52a9c99aae568056b2#info#compaction#44 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:51:46,200 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/dcf5ca8acc274437ac894b30d591c35b is 1080, key is row0001/info:/1733237476025/Put/seqid=0
2024-12-03T14:51:46,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741841_1017 (size=8296)
2024-12-03T14:51:46,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741841_1017 (size=8296)
2024-12-03T14:51:46,210 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/dcf5ca8acc274437ac894b30d591c35b as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/dcf5ca8acc274437ac894b30d591c35b
2024-12-03T14:51:46,217 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc24e7f75d95dc52a9c99aae568056b2/info of cc24e7f75d95dc52a9c99aae568056b2 into dcf5ca8acc274437ac894b30d591c35b(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:51:46,217 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for cc24e7f75d95dc52a9c99aae568056b2:

2024-12-03T14:51:46,219 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C42723%2C1733237464993.1733237506219
2024-12-03T14:51:46,224 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:46,224 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:46,224 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:46,224 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:46,225 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:46,225 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237496156 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237506219
2024-12-03T14:51:46,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39623:39623),(127.0.0.1/127.0.0.1:37949:37949)]
2024-12-03T14:51:46,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237496156 is not closed yet, will try archiving it next time
2024-12-03T14:51:46,226 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237465365 to hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/oldWALs/a5d22df9eca2%2C42723%2C1733237464993.1733237465365
2024-12-03T14:51:46,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741839_1015 (size=2520)
2024-12-03T14:51:46,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741839_1015 (size=2520)
2024-12-03T14:51:46,227 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:51:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-03T14:51:46,229 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T14:51:46,230 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T14:51:46,230 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T14:51:46,350 INFO  [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T14:51:46,350 INFO  [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T14:51:46,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-03T14:51:46,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:46,383 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing cc24e7f75d95dc52a9c99aae568056b2 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T14:51:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/df6a5c7123204447b146a34a61c2e7d5 is 1080, key is row0000/info:/1733237506218/Put/seqid=0
2024-12-03T14:51:46,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741843_1019 (size=6033)
2024-12-03T14:51:46,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741843_1019 (size=6033)
2024-12-03T14:51:46,398 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/df6a5c7123204447b146a34a61c2e7d5
2024-12-03T14:51:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/df6a5c7123204447b146a34a61c2e7d5 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/df6a5c7123204447b146a34a61c2e7d5
2024-12-03T14:51:46,408 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/df6a5c7123204447b146a34a61c2e7d5, entries=1, sequenceid=18, filesize=5.9 K
2024-12-03T14:51:46,410 INFO  [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for cc24e7f75d95dc52a9c99aae568056b2 in 26ms, sequenceid=18, compaction requested=false
2024-12-03T14:51:46,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for cc24e7f75d95dc52a9c99aae568056b2:

2024-12-03T14:51:46,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:46,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-03T14:51:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-03T14:51:46,414 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-03T14:51:46,415 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 182 msec
2024-12-03T14:51:46,417 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 188 msec
2024-12-03T14:51:46,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:46,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:46,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:47,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:47,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:47,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:48,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:48,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:48,945 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:49,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:49,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:49,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:50,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:50,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:50,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:51,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:51,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:51,673 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region cc24e7f75d95dc52a9c99aae568056b2, had cached 0 bytes from a total of 14329
2024-12-03T14:51:51,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:52,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:52,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:52,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:53,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:53,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:53,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:54,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:54,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:54,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:55,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:55,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:55,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-03T14:51:56,264 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-03T14:51:56,271 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C42723%2C1733237464993.1733237516271
2024-12-03T14:51:56,279 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,279 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,279 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,279 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,280 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,280 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237506219 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237516271
2024-12-03T14:51:56,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39623:39623),(127.0.0.1/127.0.0.1:37949:37949)]
2024-12-03T14:51:56,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237506219 is not closed yet, will try archiving it next time
2024-12-03T14:51:56,281 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/WALs/a5d22df9eca2,42723,1733237464993/a5d22df9eca2%2C42723%2C1733237464993.1733237496156 to hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/oldWALs/a5d22df9eca2%2C42723%2C1733237464993.1733237496156
2024-12-03T14:51:56,281 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-03T14:51:56,281 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:51:56,281 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:51:56,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:56,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:56,282 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-03T14:51:56,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741842_1018 (size=2026)
2024-12-03T14:51:56,282 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T14:51:56,282 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1765153706, stopped=false
2024-12-03T14:51:56,282 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,44923,1733237464952
2024-12-03T14:51:56,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741842_1018 (size=2026)
2024-12-03T14:51:56,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:51:56,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:51:56,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:56,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:56,285 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:51:56,285 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:51:56,285 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:51:56,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:56,285 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,42723,1733237464993' *****
2024-12-03T14:51:56,285 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:51:56,286 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(3091): Received CLOSE for cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:42723.
2024-12-03T14:51:56,286 DEBUG [RS:0;a5d22df9eca2:42723 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:51:56,286 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cc24e7f75d95dc52a9c99aae568056b2, disabling compactions & flushes
2024-12-03T14:51:56,286 DEBUG [RS:0;a5d22df9eca2:42723 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:56,286 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:56,286 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:51:56,286 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2. after waiting 0 ms
2024-12-03T14:51:56,286 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:51:56,287 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:51:56,287 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:56,287 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-03T14:51:56,287 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cc24e7f75d95dc52a9c99aae568056b2 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T14:51:56,289 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:51:56,289 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:51:56,292 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-03T14:51:56,292 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1325): Online Regions={cc24e7f75d95dc52a9c99aae568056b2=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2., 1588230740=hbase:meta,,1.1588230740}
2024-12-03T14:51:56,292 DEBUG [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, cc24e7f75d95dc52a9c99aae568056b2
2024-12-03T14:51:56,292 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:51:56,292 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:51:56,292 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:51:56,292 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:51:56,292 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:51:56,292 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-03T14:51:56,293 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/5f5c0f5d4ba74e6ba6c85532e00be939 is 1080, key is row0001/info:/1733237516267/Put/seqid=0
2024-12-03T14:51:56,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741845_1021 (size=6033)
2024-12-03T14:51:56,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741845_1021 (size=6033)
2024-12-03T14:51:56,302 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/5f5c0f5d4ba74e6ba6c85532e00be939
2024-12-03T14:51:56,309 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/.tmp/info/5f5c0f5d4ba74e6ba6c85532e00be939 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/5f5c0f5d4ba74e6ba6c85532e00be939
2024-12-03T14:51:56,315 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/5f5c0f5d4ba74e6ba6c85532e00be939, entries=1, sequenceid=22, filesize=5.9 K
2024-12-03T14:51:56,316 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/info/0908c01550fa4709972a2e19c6c896d2 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2./info:regioninfo/1733237466694/Put/seqid=0
2024-12-03T14:51:56,316 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for cc24e7f75d95dc52a9c99aae568056b2 in 29ms, sequenceid=22, compaction requested=true
2024-12-03T14:51:56,318 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/8e6e33a3285e46e985411d413a22ddd1, hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/77447121df43463ba451540935f25ce1, hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/21d594b14b8f4fd6848cd32f355d2544] to archive
2024-12-03T14:51:56,319 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T14:51:56,321 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/8e6e33a3285e46e985411d413a22ddd1 to hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/8e6e33a3285e46e985411d413a22ddd1
2024-12-03T14:51:56,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741846_1022 (size=7308)
2024-12-03T14:51:56,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741846_1022 (size=7308)
2024-12-03T14:51:56,322 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/info/0908c01550fa4709972a2e19c6c896d2
2024-12-03T14:51:56,323 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/77447121df43463ba451540935f25ce1 to hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/77447121df43463ba451540935f25ce1
2024-12-03T14:51:56,324 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/21d594b14b8f4fd6848cd32f355d2544 to hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/info/21d594b14b8f4fd6848cd32f355d2544
2024-12-03T14:51:56,324 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a5d22df9eca2:44923 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-03T14:51:56,325 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [8e6e33a3285e46e985411d413a22ddd1=6033, 77447121df43463ba451540935f25ce1=6033, 21d594b14b8f4fd6848cd32f355d2544=6033]
2024-12-03T14:51:56,328 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/cc24e7f75d95dc52a9c99aae568056b2/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-03T14:51:56,329 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:56,329 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cc24e7f75d95dc52a9c99aae568056b2:
Waiting for close lock at 1733237516286Running coprocessor pre-close hooks at 1733237516286Disabling compacts and flushes for region at 1733237516286Disabling writes for close at 1733237516286Obtaining lock to block concurrent updates at 1733237516287 (+1 ms)Preparing flush snapshotting stores in cc24e7f75d95dc52a9c99aae568056b2 at 1733237516287Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733237516287Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2. at 1733237516288 (+1 ms)Flushing cc24e7f75d95dc52a9c99aae568056b2/info: creating writer at 1733237516288Flushing cc24e7f75d95dc52a9c99aae568056b2/info: appending metadata at 1733237516292 (+4 ms)Flushing cc24e7f75d95dc52a9c99aae568056b2/info: closing flushed file at 1733237516292Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@407fb08c: reopening flushed file at 1733237516308 (+16 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for cc24e7f75d95dc52a9c99aae568056b2 in 29ms, sequenceid=22, compaction requested=true at 1733237516316 (+8 ms)Writing region close event to WAL at 1733237516325 (+9 ms)Running coprocessor post-close hooks at 1733237516329 (+4 ms)Closed at 1733237516329
2024-12-03T14:51:56,329 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733237465922.cc24e7f75d95dc52a9c99aae568056b2.
2024-12-03T14:51:56,340 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/ns/7e47c3dcc2054c9aaf72dd0e4f7998a4 is 43, key is default/ns:d/1733237465803/Put/seqid=0
2024-12-03T14:51:56,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741847_1023 (size=5153)
2024-12-03T14:51:56,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741847_1023 (size=5153)
2024-12-03T14:51:56,344 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/ns/7e47c3dcc2054c9aaf72dd0e4f7998a4
2024-12-03T14:51:56,362 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/table/9cde4ee3dd464fc584f74ab0c37270d2 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733237466708/Put/seqid=0
2024-12-03T14:51:56,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741848_1024 (size=5508)
2024-12-03T14:51:56,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741848_1024 (size=5508)
2024-12-03T14:51:56,367 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/table/9cde4ee3dd464fc584f74ab0c37270d2
2024-12-03T14:51:56,372 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/info/0908c01550fa4709972a2e19c6c896d2 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/info/0908c01550fa4709972a2e19c6c896d2
2024-12-03T14:51:56,377 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/info/0908c01550fa4709972a2e19c6c896d2, entries=10, sequenceid=11, filesize=7.1 K
2024-12-03T14:51:56,378 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/ns/7e47c3dcc2054c9aaf72dd0e4f7998a4 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/ns/7e47c3dcc2054c9aaf72dd0e4f7998a4
2024-12-03T14:51:56,383 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/ns/7e47c3dcc2054c9aaf72dd0e4f7998a4, entries=2, sequenceid=11, filesize=5.0 K
2024-12-03T14:51:56,384 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/.tmp/table/9cde4ee3dd464fc584f74ab0c37270d2 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/table/9cde4ee3dd464fc584f74ab0c37270d2
2024-12-03T14:51:56,388 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/table/9cde4ee3dd464fc584f74ab0c37270d2, entries=2, sequenceid=11, filesize=5.4 K
2024-12-03T14:51:56,389 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=11, compaction requested=false
2024-12-03T14:51:56,392 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-03T14:51:56,393 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:51:56,393 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:51:56,393 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237516292Running coprocessor pre-close hooks at 1733237516292Disabling compacts and flushes for region at 1733237516292Disabling writes for close at 1733237516292Obtaining lock to block concurrent updates at 1733237516292Preparing flush snapshotting stores in 1588230740 at 1733237516292Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733237516292Flushing stores of hbase:meta,,1.1588230740 at 1733237516293 (+1 ms)Flushing 1588230740/info: creating writer at 1733237516293Flushing 1588230740/info: appending metadata at 1733237516315 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733237516315Flushing 1588230740/ns: creating writer at 1733237516327 (+12 ms)Flushing 1588230740/ns: appending metadata at 1733237516339 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733237516339Flushing 1588230740/table: creating writer at 1733237516349 (+10 ms)Flushing 1588230740/table: appending metadata at 1733237516361 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733237516361Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ee3099a: reopening flushed file at 1733237516371 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@430227e: reopening flushed file at 1733237516377 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c0c82e: reopening flushed file at 1733237516383 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=11, compaction requested=false at 1733237516389 (+6 ms)Writing region close event to WAL at 1733237516390 (+1 ms)Running coprocessor post-close hooks at 1733237516393 (+3 ms)Closed at 1733237516393
2024-12-03T14:51:56,393 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T14:51:56,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:56,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:56,492 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,42723,1733237464993; all regions closed.
2024-12-03T14:51:56,493 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,493 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,493 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,493 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,493 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741834_1010 (size=3306)
2024-12-03T14:51:56,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741834_1010 (size=3306)
2024-12-03T14:51:56,502 DEBUG [RS:0;a5d22df9eca2:42723 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/oldWALs
2024-12-03T14:51:56,502 INFO  [RS:0;a5d22df9eca2:42723 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C42723%2C1733237464993.meta:.meta(num 1733237465758)
2024-12-03T14:51:56,503 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,503 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,503 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,503 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,504 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741844_1020 (size=1252)
2024-12-03T14:51:56,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741844_1020 (size=1252)
2024-12-03T14:51:56,511 DEBUG [RS:0;a5d22df9eca2:42723 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/oldWALs
2024-12-03T14:51:56,511 INFO  [RS:0;a5d22df9eca2:42723 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C42723%2C1733237464993:(num 1733237516271)
2024-12-03T14:51:56,511 DEBUG [RS:0;a5d22df9eca2:42723 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:56,511 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:51:56,511 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:51:56,511 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-03T14:51:56,511 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:51:56,511 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:51:56,512 INFO  [RS:0;a5d22df9eca2:42723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42723
2024-12-03T14:51:56,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:51:56,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,42723,1733237464993
2024-12-03T14:51:56,513 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:51:56,514 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,42723,1733237464993]
2024-12-03T14:51:56,515 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,42723,1733237464993 already deleted, retry=false
2024-12-03T14:51:56,515 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,42723,1733237464993 expired; onlineServers=0
2024-12-03T14:51:56,515 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,44923,1733237464952' *****
2024-12-03T14:51:56,515 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T14:51:56,515 INFO  [M:0;a5d22df9eca2:44923 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:51:56,515 INFO  [M:0;a5d22df9eca2:44923 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:51:56,515 DEBUG [M:0;a5d22df9eca2:44923 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T14:51:56,515 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T14:51:56,515 DEBUG [M:0;a5d22df9eca2:44923 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T14:51:56,515 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237465141 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237465141,5,FailOnTimeoutGroup]
2024-12-03T14:51:56,515 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237465141 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237465141,5,FailOnTimeoutGroup]
2024-12-03T14:51:56,515 INFO  [M:0;a5d22df9eca2:44923 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-03T14:51:56,516 INFO  [M:0;a5d22df9eca2:44923 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:51:56,516 DEBUG [M:0;a5d22df9eca2:44923 {}] master.HMaster(1795): Stopping service threads
2024-12-03T14:51:56,516 INFO  [M:0;a5d22df9eca2:44923 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T14:51:56,516 INFO  [M:0;a5d22df9eca2:44923 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:51:56,516 INFO  [M:0;a5d22df9eca2:44923 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T14:51:56,516 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T14:51:56,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T14:51:56,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:56,516 DEBUG [M:0;a5d22df9eca2:44923 {}] zookeeper.ZKUtil(347): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T14:51:56,516 WARN  [M:0;a5d22df9eca2:44923 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T14:51:56,517 INFO  [M:0;a5d22df9eca2:44923 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/.lastflushedseqids
2024-12-03T14:51:56,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741849_1025 (size=130)
2024-12-03T14:51:56,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741849_1025 (size=130)
2024-12-03T14:51:56,522 INFO  [M:0;a5d22df9eca2:44923 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-03T14:51:56,522 INFO  [M:0;a5d22df9eca2:44923 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T14:51:56,523 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:51:56,523 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:56,523 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:56,523 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:51:56,523 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:56,523 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.57 KB heapSize=54.96 KB
2024-12-03T14:51:56,535 DEBUG [M:0;a5d22df9eca2:44923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3f67370a72614a7dafbe745e9e728518 is 82, key is hbase:meta,,1/info:regioninfo/1733237465791/Put/seqid=0
2024-12-03T14:51:56,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741850_1026 (size=5672)
2024-12-03T14:51:56,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741850_1026 (size=5672)
2024-12-03T14:51:56,540 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3f67370a72614a7dafbe745e9e728518
2024-12-03T14:51:56,556 DEBUG [M:0;a5d22df9eca2:44923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b48bffc8f8104c68a59b1654286b156b is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733237466713/Put/seqid=0
2024-12-03T14:51:56,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741851_1027 (size=7820)
2024-12-03T14:51:56,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741851_1027 (size=7820)
2024-12-03T14:51:56,561 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b48bffc8f8104c68a59b1654286b156b
2024-12-03T14:51:56,564 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b48bffc8f8104c68a59b1654286b156b
2024-12-03T14:51:56,578 DEBUG [M:0;a5d22df9eca2:44923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1254b53197fd4990b56aef547299178e is 69, key is a5d22df9eca2,42723,1733237464993/rs:state/1733237465223/Put/seqid=0
2024-12-03T14:51:56,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741852_1028 (size=5156)
2024-12-03T14:51:56,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741852_1028 (size=5156)
2024-12-03T14:51:56,583 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1254b53197fd4990b56aef547299178e
2024-12-03T14:51:56,602 DEBUG [M:0;a5d22df9eca2:44923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/93a0c0d81da948b4b2730980a68a1f16 is 52, key is load_balancer_on/state:d/1733237465918/Put/seqid=0
2024-12-03T14:51:56,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741853_1029 (size=5056)
2024-12-03T14:51:56,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741853_1029 (size=5056)
2024-12-03T14:51:56,606 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/93a0c0d81da948b4b2730980a68a1f16
2024-12-03T14:51:56,611 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3f67370a72614a7dafbe745e9e728518 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3f67370a72614a7dafbe745e9e728518
2024-12-03T14:51:56,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:56,614 INFO  [RS:0;a5d22df9eca2:42723 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:51:56,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42723-0x100a0903bd90001, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:56,614 INFO  [RS:0;a5d22df9eca2:42723 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,42723,1733237464993; zookeeper connection closed.
2024-12-03T14:51:56,615 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@722a8dbd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@722a8dbd
2024-12-03T14:51:56,615 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T14:51:56,616 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3f67370a72614a7dafbe745e9e728518, entries=8, sequenceid=121, filesize=5.5 K
2024-12-03T14:51:56,617 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b48bffc8f8104c68a59b1654286b156b as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b48bffc8f8104c68a59b1654286b156b
2024-12-03T14:51:56,623 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b48bffc8f8104c68a59b1654286b156b
2024-12-03T14:51:56,623 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b48bffc8f8104c68a59b1654286b156b, entries=14, sequenceid=121, filesize=7.6 K
2024-12-03T14:51:56,624 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1254b53197fd4990b56aef547299178e as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1254b53197fd4990b56aef547299178e
2024-12-03T14:51:56,628 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1254b53197fd4990b56aef547299178e, entries=1, sequenceid=121, filesize=5.0 K
2024-12-03T14:51:56,629 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/93a0c0d81da948b4b2730980a68a1f16 as hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/93a0c0d81da948b4b2730980a68a1f16
2024-12-03T14:51:56,633 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46013/user/jenkins/test-data/9fb38dc3-717a-177f-5488-daa14f710a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/93a0c0d81da948b4b2730980a68a1f16, entries=1, sequenceid=121, filesize=4.9 K
2024-12-03T14:51:56,634 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=121, compaction requested=false
2024-12-03T14:51:56,636 INFO  [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:56,636 DEBUG [M:0;a5d22df9eca2:44923 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237516523Disabling compacts and flushes for region at 1733237516523Disabling writes for close at 1733237516523Obtaining lock to block concurrent updates at 1733237516523Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237516523Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44611, getHeapSize=56216, getOffHeapSize=0, getCellsCount=140 at 1733237516523Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237516524 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237516524Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237516535 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237516535Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237516544 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237516556 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237516556Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237516564 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237516578 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237516578Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733237516587 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733237516601 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733237516601Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5edcde8c: reopening flushed file at 1733237516610 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e527dc6: reopening flushed file at 1733237516616 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dc7c1e9: reopening flushed file at 1733237516623 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@587d48f8: reopening flushed file at 1733237516629 (+6 ms)Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=121, compaction requested=false at 1733237516634 (+5 ms)Writing region close event to WAL at 1733237516636 (+2 ms)Closed at 1733237516636
2024-12-03T14:51:56,636 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,636 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,636 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,636 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,636 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:51:56,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37447 is added to blk_1073741830_1006 (size=53008)
2024-12-03T14:51:56,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33879 is added to blk_1073741830_1006 (size=53008)
2024-12-03T14:51:56,639 INFO  [M:0;a5d22df9eca2:44923 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-03T14:51:56,639 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:51:56,639 INFO  [M:0;a5d22df9eca2:44923 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44923
2024-12-03T14:51:56,639 INFO  [M:0;a5d22df9eca2:44923 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:51:56,740 INFO  [M:0;a5d22df9eca2:44923 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:51:56,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:56,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44923-0x100a0903bd90000, quorum=127.0.0.1:54868, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:51:56,745 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d8d7f9b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:56,745 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@190ad9e7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:51:56,746 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:51:56,746 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16aeea80{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:51:56,746 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b24fbcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir/,STOPPED}
2024-12-03T14:51:56,749 WARN  [BP-1518659009-172.17.0.2-1733237464412 heartbeating to localhost/127.0.0.1:46013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:51:56,749 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:51:56,749 WARN  [BP-1518659009-172.17.0.2-1733237464412 heartbeating to localhost/127.0.0.1:46013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1518659009-172.17.0.2-1733237464412 (Datanode Uuid 3fb0bf93-3031-4140-a3e0-ed7749156b60) service to localhost/127.0.0.1:46013
2024-12-03T14:51:56,749 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:51:56,750 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data3/current/BP-1518659009-172.17.0.2-1733237464412 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:56,751 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data4/current/BP-1518659009-172.17.0.2-1733237464412 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:56,751 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:51:56,755 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d027a3b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:56,755 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b5b6aa1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:51:56,755 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:51:56,756 WARN  [BP-1518659009-172.17.0.2-1733237464412 heartbeating to localhost/127.0.0.1:46013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1518659009-172.17.0.2-1733237464412 (Datanode Uuid 7badb565-01a6-4ee7-b8fb-7e1b2bd45105) service to localhost/127.0.0.1:46013
2024-12-03T14:51:56,756 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fa328f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:51:56,756 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b3167c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir/,STOPPED}
2024-12-03T14:51:56,756 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data1/current/BP-1518659009-172.17.0.2-1733237464412 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:56,757 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/cluster_d13abf5f-35fb-cf09-8d00-e3675773fa0c/data/data2/current/BP-1518659009-172.17.0.2-1733237464412 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:51:56,759 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:51:56,765 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@232381c8{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:51:56,765 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@290f188f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:51:56,765 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:51:56,766 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5bbcc3bf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:51:56,766 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78b6351{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir/,STOPPED}
2024-12-03T14:51:56,771 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-03T14:51:56,787 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-03T14:51:56,794 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=207 (was 180)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:46013
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46013 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46013
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Command processor
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46013
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/a5d22df9eca2:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46013
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46013 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46013
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46013 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=485 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 99) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4758 (was 4654) - AvailableMemoryMB LEAK? -
2024-12-03T14:51:56,800 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=207, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=4758
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.log.dir so I do NOT create it in target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/18da1f16-77e2-2eb2-a432-9f4db1c8f1ae/hadoop.tmp.dir so I do NOT create it in target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d, deleteOnExit=true
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/test.cache.data in system properties and HBase conf
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir in system properties and HBase conf
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T14:51:56,801 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-03T14:51:56,801 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/nfs.dump.dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/java.io.tmpdir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:51:56,802 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T14:51:56,803 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T14:51:56,815 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:51:56,852 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:56,855 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:51:56,856 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:51:56,856 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:51:56,856 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:51:56,857 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:56,857 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c28d9ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:51:56,857 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75ee0a96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:51:56,946 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@8e4b628{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/java.io.tmpdir/jetty-localhost-38865-hadoop-hdfs-3_4_1-tests_jar-_-any-15895679287652807098/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:51:56,947 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7e466bdb{HTTP/1.1, (http/1.1)}{localhost:38865}
2024-12-03T14:51:56,947 INFO  [Time-limited test {}] server.Server(415): Started @240254ms
2024-12-03T14:51:56,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:56,957 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:51:57,038 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:57,040 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:51:57,041 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:51:57,041 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:51:57,041 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:51:57,043 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36eaef54{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:51:57,043 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47b62c1f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:51:57,144 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4181d37d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/java.io.tmpdir/jetty-localhost-38193-hadoop-hdfs-3_4_1-tests_jar-_-any-35944083015869879/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:57,144 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c10c155{HTTP/1.1, (http/1.1)}{localhost:38193}
2024-12-03T14:51:57,144 INFO  [Time-limited test {}] server.Server(415): Started @240451ms
2024-12-03T14:51:57,145 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:51:57,170 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:51:57,172 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:51:57,173 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:51:57,173 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:51:57,173 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T14:51:57,174 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26881465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:51:57,174 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@277e18bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:51:57,198 WARN  [Thread-1955 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data1/current/BP-1721694701-172.17.0.2-1733237516819/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:57,198 WARN  [Thread-1956 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data2/current/BP-1721694701-172.17.0.2-1733237516819/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:57,211 WARN  [Thread-1934 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:51:57,212 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ff5e26d94778951 with lease ID 0xf8df90923d1f1d45: Processing first storage report for DS-a46227b2-3a3b-4032-806b-6984107fc52e from datanode DatanodeRegistration(127.0.0.1:44863, datanodeUuid=948de6bc-b476-47b3-bdef-9e82cbcc4980, infoPort=45073, infoSecurePort=0, ipcPort=34507, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819)
2024-12-03T14:51:57,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ff5e26d94778951 with lease ID 0xf8df90923d1f1d45: from storage DS-a46227b2-3a3b-4032-806b-6984107fc52e node DatanodeRegistration(127.0.0.1:44863, datanodeUuid=948de6bc-b476-47b3-bdef-9e82cbcc4980, infoPort=45073, infoSecurePort=0, ipcPort=34507, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:57,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ff5e26d94778951 with lease ID 0xf8df90923d1f1d45: Processing first storage report for DS-5cac8a0b-8141-4e6b-b9b4-03f2341ef94f from datanode DatanodeRegistration(127.0.0.1:44863, datanodeUuid=948de6bc-b476-47b3-bdef-9e82cbcc4980, infoPort=45073, infoSecurePort=0, ipcPort=34507, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819)
2024-12-03T14:51:57,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ff5e26d94778951 with lease ID 0xf8df90923d1f1d45: from storage DS-5cac8a0b-8141-4e6b-b9b4-03f2341ef94f node DatanodeRegistration(127.0.0.1:44863, datanodeUuid=948de6bc-b476-47b3-bdef-9e82cbcc4980, infoPort=45073, infoSecurePort=0, ipcPort=34507, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:57,238 INFO  [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:51:57,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@551592b1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/java.io.tmpdir/jetty-localhost-46451-hadoop-hdfs-3_4_1-tests_jar-_-any-16040573674745838544/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:51:57,286 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7193a060{HTTP/1.1, (http/1.1)}{localhost:46451}
2024-12-03T14:51:57,286 INFO  [Time-limited test {}] server.Server(415): Started @240593ms
2024-12-03T14:51:57,287 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:51:57,353 WARN  [Thread-1981 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data3/current/BP-1721694701-172.17.0.2-1733237516819/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:57,353 WARN  [Thread-1982 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data4/current/BP-1721694701-172.17.0.2-1733237516819/current, will proceed with Du for space computation calculation, 
2024-12-03T14:51:57,372 WARN  [Thread-1970 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:51:57,373 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb41c804512f94ba8 with lease ID 0xf8df90923d1f1d46: Processing first storage report for DS-55f7d35d-674f-4d8d-ad86-3e41d2c0360d from datanode DatanodeRegistration(127.0.0.1:43195, datanodeUuid=fd8a4e44-b149-4938-856d-1a7328dc9553, infoPort=35851, infoSecurePort=0, ipcPort=40265, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819)
2024-12-03T14:51:57,374 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb41c804512f94ba8 with lease ID 0xf8df90923d1f1d46: from storage DS-55f7d35d-674f-4d8d-ad86-3e41d2c0360d node DatanodeRegistration(127.0.0.1:43195, datanodeUuid=fd8a4e44-b149-4938-856d-1a7328dc9553, infoPort=35851, infoSecurePort=0, ipcPort=40265, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:57,374 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb41c804512f94ba8 with lease ID 0xf8df90923d1f1d46: Processing first storage report for DS-9a61ed58-7a32-4488-93e4-4e2f73c03666 from datanode DatanodeRegistration(127.0.0.1:43195, datanodeUuid=fd8a4e44-b149-4938-856d-1a7328dc9553, infoPort=35851, infoSecurePort=0, ipcPort=40265, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819)
2024-12-03T14:51:57,374 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb41c804512f94ba8 with lease ID 0xf8df90923d1f1d46: from storage DS-9a61ed58-7a32-4488-93e4-4e2f73c03666 node DatanodeRegistration(127.0.0.1:43195, datanodeUuid=fd8a4e44-b149-4938-856d-1a7328dc9553, infoPort=35851, infoSecurePort=0, ipcPort=40265, storageInfo=lv=-57;cid=testClusterID;nsid=375305679;c=1733237516819), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:51:57,414 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58
2024-12-03T14:51:57,417 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/zookeeper_0, clientPort=62151, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T14:51:57,419 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62151
2024-12-03T14:51:57,420 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:57,422 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:57,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:57,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:57,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:51:57,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:51:57,432 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e with version=8
2024-12-03T14:51:57,432 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase-staging
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-03T14:51:57,434 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:51:57,435 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34805
2024-12-03T14:51:57,436 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34805 connecting to ZooKeeper ensemble=127.0.0.1:62151
2024-12-03T14:51:57,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:348050x0, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:51:57,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34805-0x100a09108d60000 connected
2024-12-03T14:51:57,448 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:57,450 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:57,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:51:57,453 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e, hbase.cluster.distributed=false
2024-12-03T14:51:57,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:51:57,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34805
2024-12-03T14:51:57,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34805
2024-12-03T14:51:57,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34805
2024-12-03T14:51:57,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34805
2024-12-03T14:51:57,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34805
2024-12-03T14:51:57,474 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:51:57,474 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:57,474 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:57,474 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:51:57,474 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:51:57,474 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:51:57,475 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:51:57,475 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:51:57,475 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40149
2024-12-03T14:51:57,476 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40149 connecting to ZooKeeper ensemble=127.0.0.1:62151
2024-12-03T14:51:57,477 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:57,478 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:57,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401490x0, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:51:57,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:401490x0, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:51:57,481 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40149-0x100a09108d60001 connected
2024-12-03T14:51:57,482 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:51:57,482 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:51:57,483 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T14:51:57,484 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:51:57,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40149
2024-12-03T14:51:57,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40149
2024-12-03T14:51:57,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40149
2024-12-03T14:51:57,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40149
2024-12-03T14:51:57,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40149
2024-12-03T14:51:57,495 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:34805
2024-12-03T14:51:57,496 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:57,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:57,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:57,497 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:57,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T14:51:57,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,498 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:51:57,499 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,34805,1733237517434 from backup master directory
2024-12-03T14:51:57,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:57,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:57,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:51:57,499 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:51:57,499 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:57,505 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/hbase.id] with ID: 3b3e8635-7785-491e-95e0-70e6ff460eb0
2024-12-03T14:51:57,505 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/.tmp/hbase.id
2024-12-03T14:51:57,511 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:51:57,511 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:51:57,511 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/.tmp/hbase.id]:[hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/hbase.id]
2024-12-03T14:51:57,522 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:57,523 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-03T14:51:57,524 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-03T14:51:57,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:51:57,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:51:57,532 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:51:57,533 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T14:51:57,533 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:51:57,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:51:57,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:51:57,544 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store
2024-12-03T14:51:57,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:51:57,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:51:57,552 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:57,552 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:51:57,552 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:57,553 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:57,553 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:51:57,553 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:57,553 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:51:57,553 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237517552Disabling compacts and flushes for region at 1733237517552Disabling writes for close at 1733237517553 (+1 ms)Writing region close event to WAL at 1733237517553Closed at 1733237517553
2024-12-03T14:51:57,554 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/.initializing
2024-12-03T14:51:57,554 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/WALs/a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:57,556 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C34805%2C1733237517434, suffix=, logDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/WALs/a5d22df9eca2,34805,1733237517434, archiveDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/oldWALs, maxLogs=10
2024-12-03T14:51:57,556 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C34805%2C1733237517434.1733237517556
2024-12-03T14:51:57,562 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/WALs/a5d22df9eca2,34805,1733237517434/a5d22df9eca2%2C34805%2C1733237517434.1733237517556
2024-12-03T14:51:57,563 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45073:45073),(127.0.0.1/127.0.0.1:35851:35851)]
2024-12-03T14:51:57,573 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:51:57,573 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:57,573 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,573 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,575 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:57,576 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T14:51:57,576 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,577 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:57,577 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:57,578 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T14:51:57,579 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,579 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:57,579 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:57,581 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T14:51:57,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,582 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:57,582 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:51:57,583 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T14:51:57,583 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,584 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:57,584 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,585 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,585 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,587 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,587 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,588 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T14:51:57,589 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:51:57,592 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:51:57,593 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=847224, jitterRate=0.07730115950107574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T14:51:57,593 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733237517573Initializing all the Stores at 1733237517574 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237517574Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517574Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517574Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517574Cleaning up temporary data from old regions at 1733237517587 (+13 ms)Region opened successfully at 1733237517593 (+6 ms)
2024-12-03T14:51:57,593 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T14:51:57,597 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16de0be4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:51:57,597 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-03T14:51:57,597 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T14:51:57,597 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T14:51:57,598 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T14:51:57,598 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T14:51:57,598 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-03T14:51:57,599 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T14:51:57,610 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T14:51:57,611 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T14:51:57,612 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-03T14:51:57,613 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T14:51:57,614 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T14:51:57,615 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-03T14:51:57,615 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T14:51:57,618 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T14:51:57,619 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-03T14:51:57,620 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T14:51:57,621 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T14:51:57,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T14:51:57,628 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T14:51:57,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:51:57,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:51:57,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,631 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,34805,1733237517434, sessionid=0x100a09108d60000, setting cluster-up flag (Was=false)
2024-12-03T14:51:57,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,636 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T14:51:57,637 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:57,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:57,645 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T14:51:57,646 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:57,650 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-03T14:51:57,655 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:57,655 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-03T14:51:57,655 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T14:51:57,656 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,34805,1733237517434
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:51:57,658 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,666 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:57,666 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-03T14:51:57,668 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,668 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T14:51:57,673 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237547673
2024-12-03T14:51:57,673 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T14:51:57,673 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T14:51:57,673 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T14:51:57,673 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T14:51:57,673 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T14:51:57,674 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T14:51:57,674 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,674 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T14:51:57,674 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T14:51:57,674 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T14:51:57,675 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T14:51:57,675 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T14:51:57,675 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237517675,5,FailOnTimeoutGroup]
2024-12-03T14:51:57,675 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237517675,5,FailOnTimeoutGroup]
2024-12-03T14:51:57,675 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,675 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T14:51:57,675 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,675 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:51:57,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:51:57,679 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-03T14:51:57,679 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e
2024-12-03T14:51:57,687 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(746): ClusterId : 3b3e8635-7785-491e-95e0-70e6ff460eb0
2024-12-03T14:51:57,687 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:51:57,688 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:51:57,689 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:51:57,690 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:51:57,690 DEBUG [RS:0;a5d22df9eca2:40149 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14be4b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:51:57,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:51:57,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:51:57,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:57,705 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:51:57,707 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:51:57,707 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,708 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:57,708 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:51:57,710 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:51:57,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,711 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:57,711 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:51:57,712 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:51:57,712 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,713 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:40149
2024-12-03T14:51:57,713 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:51:57,713 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:51:57,713 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:51:57,713 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:57,713 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:51:57,714 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,34805,1733237517434 with port=40149, startcode=1733237517474
2024-12-03T14:51:57,714 DEBUG [RS:0;a5d22df9eca2:40149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:51:57,715 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:51:57,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:57,715 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:57,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:51:57,716 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740
2024-12-03T14:51:57,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740
2024-12-03T14:51:57,718 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:51:57,718 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34805 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:57,719 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34805 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:57,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:51:57,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:51:57,720 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:51:57,721 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e
2024-12-03T14:51:57,721 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38757
2024-12-03T14:51:57,721 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:51:57,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:51:57,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:51:57,725 DEBUG [RS:0;a5d22df9eca2:40149 {}] zookeeper.ZKUtil(111): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:57,725 WARN  [RS:0;a5d22df9eca2:40149 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:51:57,725 INFO  [RS:0;a5d22df9eca2:40149 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:51:57,725 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:57,730 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,40149,1733237517474]
2024-12-03T14:51:57,733 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:51:57,734 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811929, jitterRate=0.03242132067680359}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:51:57,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733237517703Initializing all the Stores at 1733237517703Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237517704 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237517705 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517705Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237517705Cleaning up temporary data from old regions at 1733237517720 (+15 ms)Region opened successfully at 1733237517734 (+14 ms)
2024-12-03T14:51:57,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:51:57,735 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:51:57,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:51:57,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:51:57,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:51:57,735 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:51:57,737 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:51:57,737 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:51:57,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237517735Disabling compacts and flushes for region at 1733237517735Disabling writes for close at 1733237517735Writing region close event to WAL at 1733237517737 (+2 ms)Closed at 1733237517737
2024-12-03T14:51:57,739 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:57,739 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-03T14:51:57,739 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T14:51:57,741 INFO  [RS:0;a5d22df9eca2:40149 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:51:57,741 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:51:57,741 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,743 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T14:51:57,745 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:51:57,746 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:51:57,746 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,746 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,747 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,747 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,747 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:51:57,747 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:51:57,747 DEBUG [RS:0;a5d22df9eca2:40149 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:51:57,749 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,749 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,749 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,749 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,749 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,749 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,40149,1733237517474-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:51:57,771 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:51:57,772 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,40149,1733237517474-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,772 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,772 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.Replication(171): a5d22df9eca2,40149,1733237517474 started
2024-12-03T14:51:57,792 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:57,792 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,40149,1733237517474, RpcServer on a5d22df9eca2/172.17.0.2:40149, sessionid=0x100a09108d60001
2024-12-03T14:51:57,792 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:51:57,792 DEBUG [RS:0;a5d22df9eca2:40149 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:57,792 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,40149,1733237517474'
2024-12-03T14:51:57,792 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:51:57,793 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:51:57,793 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:51:57,793 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:51:57,793 DEBUG [RS:0;a5d22df9eca2:40149 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:57,793 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,40149,1733237517474'
2024-12-03T14:51:57,793 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:51:57,794 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:51:57,794 DEBUG [RS:0;a5d22df9eca2:40149 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:51:57,794 INFO  [RS:0;a5d22df9eca2:40149 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:51:57,794 INFO  [RS:0;a5d22df9eca2:40149 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:51:57,893 WARN  [a5d22df9eca2:34805 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-03T14:51:57,897 INFO  [RS:0;a5d22df9eca2:40149 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C40149%2C1733237517474, suffix=, logDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474, archiveDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/oldWALs, maxLogs=32
2024-12-03T14:51:57,897 INFO  [RS:0;a5d22df9eca2:40149 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C40149%2C1733237517474.1733237517897
2024-12-03T14:51:57,907 INFO  [RS:0;a5d22df9eca2:40149 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237517897
2024-12-03T14:51:57,916 DEBUG [RS:0;a5d22df9eca2:40149 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45073:45073),(127.0.0.1/127.0.0.1:35851:35851)]
2024-12-03T14:51:57,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:58,143 DEBUG [a5d22df9eca2:34805 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T14:51:58,144 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:58,146 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,40149,1733237517474, state=OPENING
2024-12-03T14:51:58,147 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T14:51:58,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:58,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:51:58,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:58,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:58,149 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:51:58,149 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,40149,1733237517474}]
2024-12-03T14:51:58,304 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T14:51:58,306 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43673, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T14:51:58,310 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-03T14:51:58,310 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:51:58,312 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C40149%2C1733237517474.meta, suffix=.meta, logDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474, archiveDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/oldWALs, maxLogs=32
2024-12-03T14:51:58,312 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C40149%2C1733237517474.meta.1733237518312.meta
2024-12-03T14:51:58,317 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.meta.1733237518312.meta
2024-12-03T14:51:58,323 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35851:35851),(127.0.0.1/127.0.0.1:45073:45073)]
2024-12-03T14:51:58,325 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:51:58,325 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T14:51:58,325 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T14:51:58,325 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T14:51:58,326 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T14:51:58,326 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:58,326 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-03T14:51:58,326 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-03T14:51:58,327 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:51:58,328 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:51:58,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:58,328 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:58,328 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:51:58,329 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:51:58,329 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:58,329 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:58,330 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:51:58,330 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:51:58,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:58,331 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:58,331 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:51:58,331 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:51:58,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:58,332 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:51:58,332 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:51:58,333 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740
2024-12-03T14:51:58,334 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740
2024-12-03T14:51:58,335 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:51:58,335 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:51:58,335 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:51:58,336 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:51:58,337 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=697331, jitterRate=-0.11329899728298187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:51:58,337 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-03T14:51:58,338 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733237518326Writing region info on filesystem at 1733237518326Initializing all the Stores at 1733237518327 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237518327Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237518327Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237518327Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237518327Cleaning up temporary data from old regions at 1733237518335 (+8 ms)Running coprocessor post-open hooks at 1733237518337 (+2 ms)Region opened successfully at 1733237518338 (+1 ms)
2024-12-03T14:51:58,339 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237518304
2024-12-03T14:51:58,341 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T14:51:58,341 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-03T14:51:58,342 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:58,342 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,40149,1733237517474, state=OPEN
2024-12-03T14:51:58,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:51:58,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:51:58,345 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:58,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:58,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:51:58,347 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T14:51:58,347 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,40149,1733237517474 in 196 msec
2024-12-03T14:51:58,349 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T14:51:58,349 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec
2024-12-03T14:51:58,350 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:51:58,350 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-03T14:51:58,351 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:51:58,351 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,40149,1733237517474, seqNum=-1]
2024-12-03T14:51:58,351 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:51:58,352 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46897, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:51:58,358 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 704 msec
2024-12-03T14:51:58,359 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237518359, completionTime=-1
2024-12-03T14:51:58,359 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T14:51:58,359 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237578361
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237638361
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34805,1733237517434-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34805,1733237517434-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34805,1733237517434-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:34805, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:58,361 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:58,362 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:58,363 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-03T14:51:58,365 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.866sec
2024-12-03T14:51:58,365 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T14:51:58,365 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T14:51:58,365 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T14:51:58,365 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T14:51:58,365 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T14:51:58,365 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34805,1733237517434-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:51:58,366 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34805,1733237517434-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T14:51:58,368 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-03T14:51:58,368 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T14:51:58,368 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,34805,1733237517434-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:51:58,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d3c271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:51:58,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,34805,-1 for getting cluster id
2024-12-03T14:51:58,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-03T14:51:58,389 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3b3e8635-7785-491e-95e0-70e6ff460eb0'
2024-12-03T14:51:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-03T14:51:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3b3e8635-7785-491e-95e0-70e6ff460eb0"

2024-12-03T14:51:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fefeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:51:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,34805,-1]
2024-12-03T14:51:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-03T14:51:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:51:58,392 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-03T14:51:58,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208716aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:51:58,393 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:51:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,40149,1733237517474, seqNum=-1]
2024-12-03T14:51:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:51:58,397 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:51:58,399 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:58,399 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:51:58,402 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-03T14:51:58,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-03T14:51:58,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a5d22df9eca2,34805,1733237517434
2024-12-03T14:51:58,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57b8358a
2024-12-03T14:51:58,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T14:51:58,405 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T14:51:58,406 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34805 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T14:51:58,406 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34805 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T14:51:58,406 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34805 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:51:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34805 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-03T14:51:58,409 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T14:51:58,409 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:58,409 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34805 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-03T14:51:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34805 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:51:58,411 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T14:51:58,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741835_1011 (size=381)
2024-12-03T14:51:58,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741835_1011 (size=381)
2024-12-03T14:51:58,422 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fbc3069cf266c503faf2cea6b61c8f93, NAME => 'TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e
2024-12-03T14:51:58,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:58,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:58,439 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741836_1012 (size=64)
2024-12-03T14:51:58,439 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741836_1012 (size=64)
2024-12-03T14:51:58,440 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:58,440 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing fbc3069cf266c503faf2cea6b61c8f93, disabling compactions & flushes
2024-12-03T14:51:58,440 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:51:58,440 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:51:58,440 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93. after waiting 0 ms
2024-12-03T14:51:58,440 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:51:58,440 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:51:58,440 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for fbc3069cf266c503faf2cea6b61c8f93:
Waiting for close lock at 1733237518440Disabling compacts and flushes for region at 1733237518440Disabling writes for close at 1733237518440Writing region close event to WAL at 1733237518440Closed at 1733237518440
2024-12-03T14:51:58,442 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T14:51:58,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733237518442"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237518442"}]},"ts":"1733237518442"}
2024-12-03T14:51:58,445 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-03T14:51:58,446 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T14:51:58,446 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237518446"}]},"ts":"1733237518446"}
2024-12-03T14:51:58,449 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-03T14:51:58,450 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, ASSIGN}]
2024-12-03T14:51:58,451 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, ASSIGN
2024-12-03T14:51:58,453 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, ASSIGN; state=OFFLINE, location=a5d22df9eca2,40149,1733237517474; forceNewPlan=false, retain=false
2024-12-03T14:51:58,604 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fbc3069cf266c503faf2cea6b61c8f93, regionState=OPENING, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:58,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, ASSIGN because future has completed
2024-12-03T14:51:58,607 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbc3069cf266c503faf2cea6b61c8f93, server=a5d22df9eca2,40149,1733237517474}]
2024-12-03T14:51:58,766 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:51:58,766 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fbc3069cf266c503faf2cea6b61c8f93, NAME => 'TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:51:58,766 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,767 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:51:58,767 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,767 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,769 INFO  [StoreOpener-fbc3069cf266c503faf2cea6b61c8f93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fbc3069cf266c503faf2cea6b61c8f93 
2024-12-03T14:51:58,771 INFO  [StoreOpener-fbc3069cf266c503faf2cea6b61c8f93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc3069cf266c503faf2cea6b61c8f93 columnFamilyName info
2024-12-03T14:51:58,771 DEBUG [StoreOpener-fbc3069cf266c503faf2cea6b61c8f93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:51:58,772 INFO  [StoreOpener-fbc3069cf266c503faf2cea6b61c8f93-1 {}] regionserver.HStore(327): Store=fbc3069cf266c503faf2cea6b61c8f93/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:51:58,772 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,773 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,773 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,774 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,774 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,775 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,777 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:51:58,778 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fbc3069cf266c503faf2cea6b61c8f93; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=815884, jitterRate=0.03745123744010925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T14:51:58,778 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:51:58,778 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fbc3069cf266c503faf2cea6b61c8f93:
Running coprocessor pre-open hook at 1733237518767Writing region info on filesystem at 1733237518767Initializing all the Stores at 1733237518768 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237518768Cleaning up temporary data from old regions at 1733237518774 (+6 ms)Running coprocessor post-open hooks at 1733237518778 (+4 ms)Region opened successfully at 1733237518778
2024-12-03T14:51:58,779 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., pid=6, masterSystemTime=1733237518760
2024-12-03T14:51:58,781 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:51:58,781 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:51:58,782 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fbc3069cf266c503faf2cea6b61c8f93, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:51:58,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbc3069cf266c503faf2cea6b61c8f93, server=a5d22df9eca2,40149,1733237517474 because future has completed
2024-12-03T14:51:58,789 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T14:51:58,789 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fbc3069cf266c503faf2cea6b61c8f93, server=a5d22df9eca2,40149,1733237517474 in 179 msec
2024-12-03T14:51:58,792 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T14:51:58,792 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, ASSIGN in 339 msec
2024-12-03T14:51:58,793 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T14:51:58,794 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237518793"}]},"ts":"1733237518793"}
2024-12-03T14:51:58,796 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-03T14:51:58,798 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T14:51:58,800 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 392 msec
2024-12-03T14:51:58,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:59,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:59,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:51:59,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:00,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:00,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:00,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:01,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,353 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,353 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,353 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,357 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:01,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:01,864 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:52:01,866 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,866 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,867 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,868 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,869 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,870 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,871 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,872 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,894 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:01,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:02,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-03T14:52:02,366 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-03T14:52:02,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T14:52:02,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:02,430 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:02,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:03,430 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:03,431 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:03,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T14:52:03,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-03T14:52:03,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:04,431 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:04,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:04,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:05,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:05,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:05,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:06,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:06,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:06,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:07,434 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:07,434 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:07,871 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:52:07,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,904 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:07,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:08,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:08,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34805 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-03T14:52:08,444 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-03T14:52:08,445 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-03T14:52:08,449 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-03T14:52:08,449 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:08,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., hostname=a5d22df9eca2,40149,1733237517474, seqNum=2]
2024-12-03T14:52:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:08,467 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:52:08,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/2b06008cdc484bd68fda286a9f5a5354 is 1080, key is row0001/info:/1733237528455/Put/seqid=0
2024-12-03T14:52:08,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741837_1013 (size=12509)
2024-12-03T14:52:08,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741837_1013 (size=12509)
2024-12-03T14:52:08,489 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/2b06008cdc484bd68fda286a9f5a5354
2024-12-03T14:52:08,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/2b06008cdc484bd68fda286a9f5a5354 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/2b06008cdc484bd68fda286a9f5a5354
2024-12-03T14:52:08,504 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/2b06008cdc484bd68fda286a9f5a5354, entries=7, sequenceid=11, filesize=12.2 K
2024-12-03T14:52:08,505 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for fbc3069cf266c503faf2cea6b61c8f93 in 39ms, sequenceid=11, compaction requested=false
2024-12-03T14:52:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:08,506 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-03T14:52:08,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/ae0d660f1cc54de69b4a985b8b788446 is 1080, key is row0008/info:/1733237528468/Put/seqid=0
2024-12-03T14:52:08,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741838_1014 (size=24376)
2024-12-03T14:52:08,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741838_1014 (size=24376)
2024-12-03T14:52:08,517 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/ae0d660f1cc54de69b4a985b8b788446
2024-12-03T14:52:08,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/ae0d660f1cc54de69b4a985b8b788446 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446
2024-12-03T14:52:08,528 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446, entries=18, sequenceid=32, filesize=23.8 K
2024-12-03T14:52:08,529 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=7.36 KB/7532 for fbc3069cf266c503faf2cea6b61c8f93 in 23ms, sequenceid=32, compaction requested=false
2024-12-03T14:52:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.0 K, sizeToCheck=16.0 K
2024-12-03T14:52:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446 because midkey is the same as first or last row
2024-12-03T14:52:08,960 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:09,436 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:09,437 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:09,961 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:10,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:10,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:10,525 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-03T14:52:10,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/c97d069f374e4419b5e8d435a83c46b1 is 1080, key is row0026/info:/1733237528507/Put/seqid=0
2024-12-03T14:52:10,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741839_1015 (size=13586)
2024-12-03T14:52:10,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741839_1015 (size=13586)
2024-12-03T14:52:10,537 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/c97d069f374e4419b5e8d435a83c46b1
2024-12-03T14:52:10,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/c97d069f374e4419b5e8d435a83c46b1 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/c97d069f374e4419b5e8d435a83c46b1
2024-12-03T14:52:10,550 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/c97d069f374e4419b5e8d435a83c46b1, entries=8, sequenceid=43, filesize=13.3 K
2024-12-03T14:52:10,551 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for fbc3069cf266c503faf2cea6b61c8f93 in 26ms, sequenceid=43, compaction requested=true
2024-12-03T14:52:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446 because midkey is the same as first or last row
2024-12-03T14:52:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbc3069cf266c503faf2cea6b61c8f93:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:10,552 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:10,552 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-03T14:52:10,553 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:10,553 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): fbc3069cf266c503faf2cea6b61c8f93/info is initiating minor compaction (all files)
2024-12-03T14:52:10,553 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbc3069cf266c503faf2cea6b61c8f93/info in TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:10,553 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/2b06008cdc484bd68fda286a9f5a5354, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/c97d069f374e4419b5e8d435a83c46b1] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp, totalSize=49.3 K
2024-12-03T14:52:10,554 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b06008cdc484bd68fda286a9f5a5354, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733237528455
2024-12-03T14:52:10,554 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0d660f1cc54de69b4a985b8b788446, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1733237528468
2024-12-03T14:52:10,554 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97d069f374e4419b5e8d435a83c46b1, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733237528507
2024-12-03T14:52:10,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/731a685e08dc4bae811ad6e37ae80735 is 1080, key is row0034/info:/1733237530527/Put/seqid=0
2024-12-03T14:52:10,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741840_1016 (size=17894)
2024-12-03T14:52:10,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741840_1016 (size=17894)
2024-12-03T14:52:10,564 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/731a685e08dc4bae811ad6e37ae80735
2024-12-03T14:52:10,569 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbc3069cf266c503faf2cea6b61c8f93#info#compaction#58 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:10,570 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/7e3634a0d9564bb68bce1d7986c4484e is 1080, key is row0001/info:/1733237528455/Put/seqid=0
2024-12-03T14:52:10,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/731a685e08dc4bae811ad6e37ae80735 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/731a685e08dc4bae811ad6e37ae80735
2024-12-03T14:52:10,579 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/731a685e08dc4bae811ad6e37ae80735, entries=12, sequenceid=58, filesize=17.5 K
2024-12-03T14:52:10,579 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for fbc3069cf266c503faf2cea6b61c8f93 in 27ms, sequenceid=58, compaction requested=false
2024-12-03T14:52:10,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:10,580 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.8 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,580 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,580 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446 because midkey is the same as first or last row
2024-12-03T14:52:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:10,581 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-03T14:52:10,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/af9d6b305364484da3b82ea8f50810e0 is 1080, key is row0046/info:/1733237530553/Put/seqid=0
2024-12-03T14:52:10,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741841_1017 (size=40670)
2024-12-03T14:52:10,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741841_1017 (size=40670)
2024-12-03T14:52:10,606 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/7e3634a0d9564bb68bce1d7986c4484e as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e
2024-12-03T14:52:10,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741842_1018 (size=17894)
2024-12-03T14:52:10,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741842_1018 (size=17894)
2024-12-03T14:52:10,611 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/af9d6b305364484da3b82ea8f50810e0
2024-12-03T14:52:10,613 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbc3069cf266c503faf2cea6b61c8f93/info of fbc3069cf266c503faf2cea6b61c8f93 into 7e3634a0d9564bb68bce1d7986c4484e(size=39.7 K), total size for store is 57.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:10,613 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:10,613 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., storeName=fbc3069cf266c503faf2cea6b61c8f93/info, priority=13, startTime=1733237530551; duration=0sec
2024-12-03T14:52:10,613 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,613 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,613 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e because midkey is the same as first or last row
2024-12-03T14:52:10,613 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,613 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,613 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e because midkey is the same as first or last row
2024-12-03T14:52:10,614 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,614 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,614 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e because midkey is the same as first or last row
2024-12-03T14:52:10,614 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:10,614 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbc3069cf266c503faf2cea6b61c8f93:info
2024-12-03T14:52:10,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/af9d6b305364484da3b82ea8f50810e0 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/af9d6b305364484da3b82ea8f50810e0
2024-12-03T14:52:10,625 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/af9d6b305364484da3b82ea8f50810e0, entries=12, sequenceid=73, filesize=17.5 K
2024-12-03T14:52:10,627 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for fbc3069cf266c503faf2cea6b61c8f93 in 45ms, sequenceid=73, compaction requested=true
2024-12-03T14:52:10,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:10,627 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=74.7 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,627 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,627 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e because midkey is the same as first or last row
2024-12-03T14:52:10,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbc3069cf266c503faf2cea6b61c8f93:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:10,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:10,627 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:10,631 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 76458 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:10,631 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): fbc3069cf266c503faf2cea6b61c8f93/info is initiating minor compaction (all files)
2024-12-03T14:52:10,631 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbc3069cf266c503faf2cea6b61c8f93/info in TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:10,631 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/731a685e08dc4bae811ad6e37ae80735, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/af9d6b305364484da3b82ea8f50810e0] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp, totalSize=74.7 K
2024-12-03T14:52:10,632 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e3634a0d9564bb68bce1d7986c4484e, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733237528455
2024-12-03T14:52:10,632 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731a685e08dc4bae811ad6e37ae80735, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733237530527
2024-12-03T14:52:10,633 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting af9d6b305364484da3b82ea8f50810e0, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733237530553
2024-12-03T14:52:10,651 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbc3069cf266c503faf2cea6b61c8f93#info#compaction#60 average throughput is 29.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:10,652 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/bd5102a4b1e54e46b47cde3507217693 is 1080, key is row0001/info:/1733237528455/Put/seqid=0
2024-12-03T14:52:10,657 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741843_1019 (size=66689)
2024-12-03T14:52:10,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741843_1019 (size=66689)
2024-12-03T14:52:10,664 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/bd5102a4b1e54e46b47cde3507217693 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693
2024-12-03T14:52:10,671 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbc3069cf266c503faf2cea6b61c8f93/info of fbc3069cf266c503faf2cea6b61c8f93 into bd5102a4b1e54e46b47cde3507217693(size=65.1 K), total size for store is 65.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:10,671 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:10,671 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., storeName=fbc3069cf266c503faf2cea6b61c8f93/info, priority=13, startTime=1733237530627; duration=0sec
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.1 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693 because midkey is the same as first or last row
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.1 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693 because midkey is the same as first or last row
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.1 K, sizeToCheck=16.0 K
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693 because midkey is the same as first or last row
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:10,672 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbc3069cf266c503faf2cea6b61c8f93:info
2024-12-03T14:52:10,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:11,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:11,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:11,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:12,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:12,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,604 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-03T14:52:12,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/3f24deab49b047908a7542fcae5860ae is 1080, key is row0058/info:/1733237530583/Put/seqid=0
2024-12-03T14:52:12,616 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741844_1020 (size=13586)
2024-12-03T14:52:12,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741844_1020 (size=13586)
2024-12-03T14:52:12,617 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/3f24deab49b047908a7542fcae5860ae
2024-12-03T14:52:12,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/3f24deab49b047908a7542fcae5860ae as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/3f24deab49b047908a7542fcae5860ae
2024-12-03T14:52:12,630 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/3f24deab49b047908a7542fcae5860ae, entries=8, sequenceid=86, filesize=13.3 K
2024-12-03T14:52:12,631 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for fbc3069cf266c503faf2cea6b61c8f93 in 27ms, sequenceid=86, compaction requested=false
2024-12-03T14:52:12,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:12,631 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.4 K, sizeToCheck=16.0 K
2024-12-03T14:52:12,631 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:12,631 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693 because midkey is the same as first or last row
2024-12-03T14:52:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,633 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-03T14:52:12,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/07e1cbb668204b9b8c1fcae89dd78efb is 1080, key is row0066/info:/1733237532606/Put/seqid=0
2024-12-03T14:52:12,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741845_1021 (size=17894)
2024-12-03T14:52:12,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741845_1021 (size=17894)
2024-12-03T14:52:12,642 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/07e1cbb668204b9b8c1fcae89dd78efb
2024-12-03T14:52:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/07e1cbb668204b9b8c1fcae89dd78efb as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/07e1cbb668204b9b8c1fcae89dd78efb
2024-12-03T14:52:12,652 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/07e1cbb668204b9b8c1fcae89dd78efb, entries=12, sequenceid=101, filesize=17.5 K
2024-12-03T14:52:12,653 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for fbc3069cf266c503faf2cea6b61c8f93 in 20ms, sequenceid=101, compaction requested=true
2024-12-03T14:52:12,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:12,653 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=95.9 K, sizeToCheck=16.0 K
2024-12-03T14:52:12,653 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:12,653 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693 because midkey is the same as first or last row
2024-12-03T14:52:12,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbc3069cf266c503faf2cea6b61c8f93:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:12,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:12,653 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,654 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 98169 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:12,654 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): fbc3069cf266c503faf2cea6b61c8f93/info is initiating minor compaction (all files)
2024-12-03T14:52:12,654 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-03T14:52:12,654 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbc3069cf266c503faf2cea6b61c8f93/info in TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:12,654 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/3f24deab49b047908a7542fcae5860ae, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/07e1cbb668204b9b8c1fcae89dd78efb] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp, totalSize=95.9 K
2024-12-03T14:52:12,655 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5102a4b1e54e46b47cde3507217693, keycount=57, bloomtype=ROW, size=65.1 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733237528455
2024-12-03T14:52:12,655 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f24deab49b047908a7542fcae5860ae, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733237530583
2024-12-03T14:52:12,655 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e1cbb668204b9b8c1fcae89dd78efb, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733237532606
2024-12-03T14:52:12,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/e4c26fec8329454ca7a76e30b5219545 is 1080, key is row0078/info:/1733237532634/Put/seqid=0
2024-12-03T14:52:12,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741846_1022 (size=17894)
2024-12-03T14:52:12,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741846_1022 (size=17894)
2024-12-03T14:52:12,664 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/e4c26fec8329454ca7a76e30b5219545
2024-12-03T14:52:12,669 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbc3069cf266c503faf2cea6b61c8f93#info#compaction#64 average throughput is 26.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:12,670 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/a497216126c14c088fc6d929804b1bfb is 1080, key is row0001/info:/1733237528455/Put/seqid=0
2024-12-03T14:52:12,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/e4c26fec8329454ca7a76e30b5219545 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/e4c26fec8329454ca7a76e30b5219545
2024-12-03T14:52:12,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741847_1023 (size=88408)
2024-12-03T14:52:12,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741847_1023 (size=88408)
2024-12-03T14:52:12,678 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/e4c26fec8329454ca7a76e30b5219545, entries=12, sequenceid=116, filesize=17.5 K
2024-12-03T14:52:12,679 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/a497216126c14c088fc6d929804b1bfb as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb
2024-12-03T14:52:12,679 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for fbc3069cf266c503faf2cea6b61c8f93 in 25ms, sequenceid=116, compaction requested=false
2024-12-03T14:52:12,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:12,679 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=113.3 K, sizeToCheck=16.0 K
2024-12-03T14:52:12,679 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:12,679 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693 because midkey is the same as first or last row
2024-12-03T14:52:12,685 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbc3069cf266c503faf2cea6b61c8f93/info of fbc3069cf266c503faf2cea6b61c8f93 into a497216126c14c088fc6d929804b1bfb(size=86.3 K), total size for store is 103.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:12,685 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbc3069cf266c503faf2cea6b61c8f93:

2024-12-03T14:52:12,685 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., storeName=fbc3069cf266c503faf2cea6b61c8f93/info, priority=13, startTime=1733237532653; duration=0sec
2024-12-03T14:52:12,685 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K
2024-12-03T14:52:12,685 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:12,686 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K
2024-12-03T14:52:12,686 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:12,686 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K
2024-12-03T14:52:12,686 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T14:52:12,687 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:12,687 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:12,687 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbc3069cf266c503faf2cea6b61c8f93:info
2024-12-03T14:52:12,688 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34805 {}] assignment.AssignmentManager(1363): Split request from a5d22df9eca2,40149,1733237517474, parent={ENCODED => fbc3069cf266c503faf2cea6b61c8f93, NAME => 'TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-03T14:52:12,693 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34805 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:12,696 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34805 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fbc3069cf266c503faf2cea6b61c8f93, daughterA=85fe1f7ea138f1a0fc13ce67fd291276, daughterB=3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:12,696 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fbc3069cf266c503faf2cea6b61c8f93, daughterA=85fe1f7ea138f1a0fc13ce67fd291276, daughterB=3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:12,697 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fbc3069cf266c503faf2cea6b61c8f93, daughterA=85fe1f7ea138f1a0fc13ce67fd291276, daughterB=3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:12,697 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fbc3069cf266c503faf2cea6b61c8f93, daughterA=85fe1f7ea138f1a0fc13ce67fd291276, daughterB=3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:12,702 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, UNASSIGN}]
2024-12-03T14:52:12,703 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, UNASSIGN
2024-12-03T14:52:12,705 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=fbc3069cf266c503faf2cea6b61c8f93, regionState=CLOSING, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:12,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, UNASSIGN because future has completed
2024-12-03T14:52:12,707 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-03T14:52:12,708 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbc3069cf266c503faf2cea6b61c8f93, server=a5d22df9eca2,40149,1733237517474}]
2024-12-03T14:52:12,868 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,868 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-03T14:52:12,869 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing fbc3069cf266c503faf2cea6b61c8f93, disabling compactions & flushes
2024-12-03T14:52:12,869 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:12,870 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:12,870 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93. after waiting 0 ms
2024-12-03T14:52:12,870 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:12,870 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing fbc3069cf266c503faf2cea6b61c8f93 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:52:12,878 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/23e44889b2b948218eb99e4d42a88691 is 1080, key is row0090/info:/1733237532656/Put/seqid=0
2024-12-03T14:52:12,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741848_1024 (size=12509)
2024-12-03T14:52:12,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741848_1024 (size=12509)
2024-12-03T14:52:12,887 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/23e44889b2b948218eb99e4d42a88691
2024-12-03T14:52:12,893 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/.tmp/info/23e44889b2b948218eb99e4d42a88691 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/23e44889b2b948218eb99e4d42a88691
2024-12-03T14:52:12,898 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/23e44889b2b948218eb99e4d42a88691, entries=7, sequenceid=127, filesize=12.2 K
2024-12-03T14:52:12,899 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fbc3069cf266c503faf2cea6b61c8f93 in 28ms, sequenceid=127, compaction requested=true
2024-12-03T14:52:12,900 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/2b06008cdc484bd68fda286a9f5a5354, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/c97d069f374e4419b5e8d435a83c46b1, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/731a685e08dc4bae811ad6e37ae80735, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/af9d6b305364484da3b82ea8f50810e0, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/3f24deab49b047908a7542fcae5860ae, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/07e1cbb668204b9b8c1fcae89dd78efb] to archive
2024-12-03T14:52:12,901 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T14:52:12,903 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/2b06008cdc484bd68fda286a9f5a5354 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/2b06008cdc484bd68fda286a9f5a5354
2024-12-03T14:52:12,904 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/ae0d660f1cc54de69b4a985b8b788446
2024-12-03T14:52:12,906 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/7e3634a0d9564bb68bce1d7986c4484e
2024-12-03T14:52:12,907 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/c97d069f374e4419b5e8d435a83c46b1 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/c97d069f374e4419b5e8d435a83c46b1
2024-12-03T14:52:12,909 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/731a685e08dc4bae811ad6e37ae80735 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/731a685e08dc4bae811ad6e37ae80735
2024-12-03T14:52:12,910 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/bd5102a4b1e54e46b47cde3507217693
2024-12-03T14:52:12,912 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/af9d6b305364484da3b82ea8f50810e0 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/af9d6b305364484da3b82ea8f50810e0
2024-12-03T14:52:12,913 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/3f24deab49b047908a7542fcae5860ae to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/3f24deab49b047908a7542fcae5860ae
2024-12-03T14:52:12,915 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/07e1cbb668204b9b8c1fcae89dd78efb to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/07e1cbb668204b9b8c1fcae89dd78efb
2024-12-03T14:52:12,920 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-12-03T14:52:12,921 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.
2024-12-03T14:52:12,921 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for fbc3069cf266c503faf2cea6b61c8f93:
Waiting for close lock at 1733237532869Running coprocessor pre-close hooks at 1733237532869Disabling compacts and flushes for region at 1733237532869Disabling writes for close at 1733237532870 (+1 ms)Obtaining lock to block concurrent updates at 1733237532870Preparing flush snapshotting stores in fbc3069cf266c503faf2cea6b61c8f93 at 1733237532870Finished memstore snapshotting TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., syncing WAL and waiting on mvcc, flushsize=dataSize=7532, getHeapSize=8304, getOffHeapSize=0, getCellsCount=7 at 1733237532870Flushing stores of TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93. at 1733237532872 (+2 ms)Flushing fbc3069cf266c503faf2cea6b61c8f93/info: creating writer at 1733237532872Flushing fbc3069cf266c503faf2cea6b61c8f93/info: appending metadata at 1733237532877 (+5 ms)Flushing fbc3069cf266c503faf2cea6b61c8f93/info: closing flushed file at 1733237532877Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a9adda2: reopening flushed file at 1733237532892 (+15 ms)Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fbc3069cf266c503faf2cea6b61c8f93 in 28ms, sequenceid=127, compaction requested=true at 1733237532899 (+7 ms)Writing region close event to WAL at 1733237532917 (+18 ms)Running coprocessor post-close hooks at 1733237532921 (+4 ms)Closed at 1733237532921
2024-12-03T14:52:12,924 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,924 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=fbc3069cf266c503faf2cea6b61c8f93, regionState=CLOSED
2024-12-03T14:52:12,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbc3069cf266c503faf2cea6b61c8f93, server=a5d22df9eca2,40149,1733237517474 because future has completed
2024-12-03T14:52:12,929 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-12-03T14:52:12,929 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure fbc3069cf266c503faf2cea6b61c8f93, server=a5d22df9eca2,40149,1733237517474 in 220 msec
2024-12-03T14:52:12,931 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-03T14:52:12,931 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbc3069cf266c503faf2cea6b61c8f93, UNASSIGN in 227 msec
2024-12-03T14:52:12,939 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:12,941 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=fbc3069cf266c503faf2cea6b61c8f93, threads=3
2024-12-03T14:52:12,943 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb for region: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,943 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/23e44889b2b948218eb99e4d42a88691 for region: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,943 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/e4c26fec8329454ca7a76e30b5219545 for region: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,952 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/e4c26fec8329454ca7a76e30b5219545, top=true
2024-12-03T14:52:12,953 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/23e44889b2b948218eb99e4d42a88691, top=true
2024-12-03T14:52:12,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741849_1025 (size=27)
2024-12-03T14:52:12,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741849_1025 (size=27)
2024-12-03T14:52:12,963 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:12,964 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545 for child: 3fa7822ea36b9984e8dc3f635eee83d8, parent: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,964 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691 for child: 3fa7822ea36b9984e8dc3f635eee83d8, parent: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,964 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/e4c26fec8329454ca7a76e30b5219545 for region: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,965 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/23e44889b2b948218eb99e4d42a88691 for region: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741850_1026 (size=27)
2024-12-03T14:52:12,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741850_1026 (size=27)
2024-12-03T14:52:12,969 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb for region: fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:12,971 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region fbc3069cf266c503faf2cea6b61c8f93 Daughter A: [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93] storefiles, Daughter B: [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545] storefiles.
2024-12-03T14:52:12,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741851_1027 (size=71)
2024-12-03T14:52:12,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741851_1027 (size=71)
2024-12-03T14:52:12,980 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:12,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741852_1028 (size=71)
2024-12-03T14:52:12,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741852_1028 (size=71)
2024-12-03T14:52:12,994 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:13,003 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-03T14:52:13,005 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-03T14:52:13,007 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733237533006"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733237533006"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733237533006"}]},"ts":"1733237533006"}
2024-12-03T14:52:13,007 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733237533006"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237533006"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733237533006"}]},"ts":"1733237533006"}
2024-12-03T14:52:13,007 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733237533006"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237533006"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733237533006"}]},"ts":"1733237533006"}
2024-12-03T14:52:13,022 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=85fe1f7ea138f1a0fc13ce67fd291276, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fa7822ea36b9984e8dc3f635eee83d8, ASSIGN}]
2024-12-03T14:52:13,024 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=85fe1f7ea138f1a0fc13ce67fd291276, ASSIGN
2024-12-03T14:52:13,024 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fa7822ea36b9984e8dc3f635eee83d8, ASSIGN
2024-12-03T14:52:13,024 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=85fe1f7ea138f1a0fc13ce67fd291276, ASSIGN; state=SPLITTING_NEW, location=a5d22df9eca2,40149,1733237517474; forceNewPlan=false, retain=false
2024-12-03T14:52:13,024 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fa7822ea36b9984e8dc3f635eee83d8, ASSIGN; state=SPLITTING_NEW, location=a5d22df9eca2,40149,1733237517474; forceNewPlan=false, retain=false
2024-12-03T14:52:13,175 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=85fe1f7ea138f1a0fc13ce67fd291276, regionState=OPENING, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:13,175 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3fa7822ea36b9984e8dc3f635eee83d8, regionState=OPENING, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:13,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fa7822ea36b9984e8dc3f635eee83d8, ASSIGN because future has completed
2024-12-03T14:52:13,181 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fa7822ea36b9984e8dc3f635eee83d8, server=a5d22df9eca2,40149,1733237517474}]
2024-12-03T14:52:13,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=85fe1f7ea138f1a0fc13ce67fd291276, ASSIGN because future has completed
2024-12-03T14:52:13,183 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85fe1f7ea138f1a0fc13ce67fd291276, server=a5d22df9eca2,40149,1733237517474}]
2024-12-03T14:52:13,339 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:13,339 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fa7822ea36b9984e8dc3f635eee83d8, NAME => 'TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-03T14:52:13,340 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,340 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:52:13,340 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,340 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,342 INFO  [StoreOpener-3fa7822ea36b9984e8dc3f635eee83d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3fa7822ea36b9984e8dc3f635eee83d8 
2024-12-03T14:52:13,343 INFO  [StoreOpener-3fa7822ea36b9984e8dc3f635eee83d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fa7822ea36b9984e8dc3f635eee83d8 columnFamilyName info
2024-12-03T14:52:13,343 DEBUG [StoreOpener-3fa7822ea36b9984e8dc3f635eee83d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:13,356 DEBUG [StoreOpener-3fa7822ea36b9984e8dc3f635eee83d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691
2024-12-03T14:52:13,360 DEBUG [StoreOpener-3fa7822ea36b9984e8dc3f635eee83d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545
2024-12-03T14:52:13,366 DEBUG [StoreOpener-3fa7822ea36b9984e8dc3f635eee83d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93->hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb-top
2024-12-03T14:52:13,367 INFO  [StoreOpener-3fa7822ea36b9984e8dc3f635eee83d8-1 {}] regionserver.HStore(327): Store=3fa7822ea36b9984e8dc3f635eee83d8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:52:13,367 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,368 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,369 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,370 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,370 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,371 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,372 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3fa7822ea36b9984e8dc3f635eee83d8; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=849649, jitterRate=0.08038538694381714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T14:52:13,372 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:13,373 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3fa7822ea36b9984e8dc3f635eee83d8:
Running coprocessor pre-open hook at 1733237533340Writing region info on filesystem at 1733237533340Initializing all the Stores at 1733237533341 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237533341Cleaning up temporary data from old regions at 1733237533370 (+29 ms)Running coprocessor post-open hooks at 1733237533372 (+2 ms)Region opened successfully at 1733237533373 (+1 ms)
2024-12-03T14:52:13,374 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., pid=12, masterSystemTime=1733237533335
2024-12-03T14:52:13,374 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:13,374 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:13,374 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:13,375 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:13,375 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:13,375 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:13,376 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93->hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb-top, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=116.0 K
2024-12-03T14:52:13,376 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:13,376 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:13,376 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93, keycount=38, bloomtype=ROW, size=86.3 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733237528455
2024-12-03T14:52:13,376 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:13,377 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 85fe1f7ea138f1a0fc13ce67fd291276, NAME => 'TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-03T14:52:13,377 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733237532634
2024-12-03T14:52:13,377 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,377 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3fa7822ea36b9984e8dc3f635eee83d8, regionState=OPEN, openSeqNum=131, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:13,377 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:52:13,377 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,377 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,377 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733237532656
2024-12-03T14:52:13,378 INFO  [StoreOpener-85fe1f7ea138f1a0fc13ce67fd291276-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 85fe1f7ea138f1a0fc13ce67fd291276 
2024-12-03T14:52:13,379 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-03T14:52:13,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-03T14:52:13,379 INFO  [StoreOpener-85fe1f7ea138f1a0fc13ce67fd291276-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85fe1f7ea138f1a0fc13ce67fd291276 columnFamilyName info
2024-12-03T14:52:13,379 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-12-03T14:52:13,379 DEBUG [StoreOpener-85fe1f7ea138f1a0fc13ce67fd291276-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:13,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fa7822ea36b9984e8dc3f635eee83d8, server=a5d22df9eca2,40149,1733237517474 because future has completed
2024-12-03T14:52:13,383 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-03T14:52:13,383 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3fa7822ea36b9984e8dc3f635eee83d8, server=a5d22df9eca2,40149,1733237517474 in 199 msec
2024-12-03T14:52:13,385 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fa7822ea36b9984e8dc3f635eee83d8, ASSIGN in 361 msec
2024-12-03T14:52:13,389 DEBUG [StoreOpener-85fe1f7ea138f1a0fc13ce67fd291276-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93->hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb-bottom
2024-12-03T14:52:13,389 INFO  [StoreOpener-85fe1f7ea138f1a0fc13ce67fd291276-1 {}] regionserver.HStore(327): Store=85fe1f7ea138f1a0fc13ce67fd291276/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:52:13,389 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,390 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,391 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,392 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,392 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,393 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,394 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 85fe1f7ea138f1a0fc13ce67fd291276; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=757070, jitterRate=-0.037336304783821106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T14:52:13,394 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:13,394 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 85fe1f7ea138f1a0fc13ce67fd291276:
Running coprocessor pre-open hook at 1733237533377Writing region info on filesystem at 1733237533377Initializing all the Stores at 1733237533378 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237533378Cleaning up temporary data from old regions at 1733237533392 (+14 ms)Running coprocessor post-open hooks at 1733237533394 (+2 ms)Region opened successfully at 1733237533394
2024-12-03T14:52:13,395 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276., pid=13, masterSystemTime=1733237533335
2024-12-03T14:52:13,395 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 85fe1f7ea138f1a0fc13ce67fd291276:info, priority=-2147483648, current under compaction store size is 2
2024-12-03T14:52:13,395 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:13,395 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-03T14:52:13,396 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:13,396 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HStore(1541): 85fe1f7ea138f1a0fc13ce67fd291276/info is initiating minor compaction (all files)
2024-12-03T14:52:13,396 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85fe1f7ea138f1a0fc13ce67fd291276/info in TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:13,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/info/be358568636342848a5cc22174380dfe is 193, key is TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8./info:regioninfo/1733237533377/Put/seqid=0
2024-12-03T14:52:13,396 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93->hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb-bottom] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/.tmp, totalSize=86.3 K
2024-12-03T14:52:13,397 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] compactions.Compactor(225): Compacting a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93, keycount=38, bloomtype=ROW, size=86.3 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733237528455
2024-12-03T14:52:13,399 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:13,399 INFO  [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:13,399 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=85fe1f7ea138f1a0fc13ce67fd291276, regionState=OPEN, openSeqNum=131, regionLocation=a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:13,400 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#67 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:13,401 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/7f4a81bb51d3417e81679a89815ce1d5 is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:13,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85fe1f7ea138f1a0fc13ce67fd291276, server=a5d22df9eca2,40149,1733237517474 because future has completed
2024-12-03T14:52:13,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741853_1029 (size=9882)
2024-12-03T14:52:13,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741853_1029 (size=9882)
2024-12-03T14:52:13,407 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/info/be358568636342848a5cc22174380dfe
2024-12-03T14:52:13,409 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-12-03T14:52:13,409 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 85fe1f7ea138f1a0fc13ce67fd291276, server=a5d22df9eca2,40149,1733237517474 in 223 msec
2024-12-03T14:52:13,412 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-12-03T14:52:13,412 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=85fe1f7ea138f1a0fc13ce67fd291276, ASSIGN in 387 msec
2024-12-03T14:52:13,422 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fbc3069cf266c503faf2cea6b61c8f93, daughterA=85fe1f7ea138f1a0fc13ce67fd291276, daughterB=3fa7822ea36b9984e8dc3f635eee83d8 in 719 msec
2024-12-03T14:52:13,427 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85fe1f7ea138f1a0fc13ce67fd291276#info#compaction#68 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:13,428 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/.tmp/info/ceb190d532964285b1b1190618b349dc is 1080, key is row0001/info:/1733237528455/Put/seqid=0
2024-12-03T14:52:13,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741854_1030 (size=42984)
2024-12-03T14:52:13,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741854_1030 (size=42984)
2024-12-03T14:52:13,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741855_1031 (size=70862)
2024-12-03T14:52:13,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741855_1031 (size=70862)
2024-12-03T14:52:13,436 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/7f4a81bb51d3417e81679a89815ce1d5 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7f4a81bb51d3417e81679a89815ce1d5
2024-12-03T14:52:13,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/ns/48390e79e8024e12ac1eab92f6c244eb is 43, key is default/ns:d/1733237518353/Put/seqid=0
2024-12-03T14:52:13,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:13,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:13,441 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/.tmp/info/ceb190d532964285b1b1190618b349dc as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/info/ceb190d532964285b1b1190618b349dc
2024-12-03T14:52:13,442 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into 7f4a81bb51d3417e81679a89815ce1d5(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:13,443 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:13,443 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237533374; duration=0sec
2024-12-03T14:52:13,443 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:13,443 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:13,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741856_1032 (size=5153)
2024-12-03T14:52:13,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741856_1032 (size=5153)
2024-12-03T14:52:13,445 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/ns/48390e79e8024e12ac1eab92f6c244eb
2024-12-03T14:52:13,448 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 85fe1f7ea138f1a0fc13ce67fd291276/info of 85fe1f7ea138f1a0fc13ce67fd291276 into ceb190d532964285b1b1190618b349dc(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:13,448 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85fe1f7ea138f1a0fc13ce67fd291276:

2024-12-03T14:52:13,448 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276., storeName=85fe1f7ea138f1a0fc13ce67fd291276/info, priority=15, startTime=1733237533395; duration=0sec
2024-12-03T14:52:13,448 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:13,448 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85fe1f7ea138f1a0fc13ce67fd291276:info
2024-12-03T14:52:13,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/table/3fd898a8b469446fb5b880479b691bfd is 65, key is TestLogRolling-testLogRolling/table:state/1733237518793/Put/seqid=0
2024-12-03T14:52:13,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741857_1033 (size=5340)
2024-12-03T14:52:13,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741857_1033 (size=5340)
2024-12-03T14:52:13,467 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/table/3fd898a8b469446fb5b880479b691bfd
2024-12-03T14:52:13,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/info/be358568636342848a5cc22174380dfe as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/info/be358568636342848a5cc22174380dfe
2024-12-03T14:52:13,478 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/info/be358568636342848a5cc22174380dfe, entries=30, sequenceid=17, filesize=9.7 K
2024-12-03T14:52:13,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/ns/48390e79e8024e12ac1eab92f6c244eb as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/ns/48390e79e8024e12ac1eab92f6c244eb
2024-12-03T14:52:13,484 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/ns/48390e79e8024e12ac1eab92f6c244eb, entries=2, sequenceid=17, filesize=5.0 K
2024-12-03T14:52:13,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/table/3fd898a8b469446fb5b880479b691bfd as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/table/3fd898a8b469446fb5b880479b691bfd
2024-12-03T14:52:13,491 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/table/3fd898a8b469446fb5b880479b691bfd, entries=2, sequenceid=17, filesize=5.2 K
2024-12-03T14:52:13,492 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 113ms, sequenceid=17, compaction requested=false
2024-12-03T14:52:13,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-03T14:52:13,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:14,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:14,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:41344 deadline: 1733237544672, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93. is not online on a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., hostname=a5d22df9eca2,40149,1733237517474, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., hostname=a5d22df9eca2,40149,1733237517474, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93. is not online on a5d22df9eca2,40149,1733237517474
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-03T14:52:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., hostname=a5d22df9eca2,40149,1733237517474, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93. is not online on a5d22df9eca2,40149,1733237517474
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-03T14:52:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733237518406.fbc3069cf266c503faf2cea6b61c8f93., hostname=a5d22df9eca2,40149,1733237517474, seqNum=2 from cache
2024-12-03T14:52:14,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:15,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:15,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:15,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:16,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:16,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:16,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:17,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:17,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:17,922 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,922 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,923 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,926 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,926 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,950 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,950 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,950 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,952 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:17,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:18,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:18,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:18,457 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T14:52:18,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,462 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,462 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,480 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,480 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,481 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,481 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,481 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,481 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,483 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,484 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,484 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,485 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T14:52:18,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:19,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:19,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:19,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:20,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:20,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:20,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:21,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:21,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:21,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:22,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:22,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:22,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:23,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:23,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:23,972 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:24,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:24,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:24,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., hostname=a5d22df9eca2,40149,1733237517474, seqNum=131]
2024-12-03T14:52:24,826 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:52:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:24,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/f770c62cc32f46be93590b3a562b752a is 1080, key is row0097/info:/1733237544805/Put/seqid=0
2024-12-03T14:52:24,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741858_1034 (size=12516)
2024-12-03T14:52:24,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741858_1034 (size=12516)
2024-12-03T14:52:24,871 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/f770c62cc32f46be93590b3a562b752a
2024-12-03T14:52:24,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/f770c62cc32f46be93590b3a562b752a as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/f770c62cc32f46be93590b3a562b752a
2024-12-03T14:52:24,889 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/f770c62cc32f46be93590b3a562b752a, entries=7, sequenceid=141, filesize=12.2 K
2024-12-03T14:52:24,891 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 3fa7822ea36b9984e8dc3f635eee83d8 in 65ms, sequenceid=141, compaction requested=false
2024-12-03T14:52:24,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:24,893 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-12-03T14:52:24,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/af16678f8b8e4f8da42821b4f8cd340d is 1080, key is row0104/info:/1733237544830/Put/seqid=0
2024-12-03T14:52:24,918 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741859_1035 (size=25472)
2024-12-03T14:52:24,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741859_1035 (size=25472)
2024-12-03T14:52:24,919 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/af16678f8b8e4f8da42821b4f8cd340d
2024-12-03T14:52:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/af16678f8b8e4f8da42821b4f8cd340d as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/af16678f8b8e4f8da42821b4f8cd340d
2024-12-03T14:52:24,931 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/af16678f8b8e4f8da42821b4f8cd340d, entries=19, sequenceid=163, filesize=24.9 K
2024-12-03T14:52:24,932 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for 3fa7822ea36b9984e8dc3f635eee83d8 in 39ms, sequenceid=163, compaction requested=true
2024-12-03T14:52:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:24,932 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:24,934 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 80972 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:24,934 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:24,934 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:24,934 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7f4a81bb51d3417e81679a89815ce1d5, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/f770c62cc32f46be93590b3a562b752a, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/af16678f8b8e4f8da42821b4f8cd340d] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=79.1 K
2024-12-03T14:52:24,934 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4a81bb51d3417e81679a89815ce1d5, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733237530593
2024-12-03T14:52:24,934 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting f770c62cc32f46be93590b3a562b752a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733237544805
2024-12-03T14:52:24,935 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting af16678f8b8e4f8da42821b4f8cd340d, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733237544830
2024-12-03T14:52:24,954 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#73 average throughput is 62.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:24,955 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/a6494aedf4114a67b04a4213933111e2 is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:24,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741860_1036 (size=71186)
2024-12-03T14:52:24,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741860_1036 (size=71186)
2024-12-03T14:52:24,967 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/a6494aedf4114a67b04a4213933111e2 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a6494aedf4114a67b04a4213933111e2
2024-12-03T14:52:24,972 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:24,973 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into a6494aedf4114a67b04a4213933111e2(size=69.5 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:24,973 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:24,973 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237544932; duration=0sec
2024-12-03T14:52:24,973 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:24,973 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:25,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:25,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:25,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:26,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:26,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:26,916 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:52:26,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/fc95ca3267814074a6c44dfd0c77e2e3 is 1080, key is row0123/info:/1733237544895/Put/seqid=0
2024-12-03T14:52:26,926 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741861_1037 (size=12516)
2024-12-03T14:52:26,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741861_1037 (size=12516)
2024-12-03T14:52:26,927 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/fc95ca3267814074a6c44dfd0c77e2e3
2024-12-03T14:52:26,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/fc95ca3267814074a6c44dfd0c77e2e3 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/fc95ca3267814074a6c44dfd0c77e2e3
2024-12-03T14:52:26,946 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/fc95ca3267814074a6c44dfd0c77e2e3, entries=7, sequenceid=174, filesize=12.2 K
2024-12-03T14:52:26,946 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 3fa7822ea36b9984e8dc3f635eee83d8 in 30ms, sequenceid=174, compaction requested=false
2024-12-03T14:52:26,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:26,948 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-03T14:52:26,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/c60b4ebafeb9458d836c60f1bc7a1735 is 1080, key is row0130/info:/1733237546918/Put/seqid=0
2024-12-03T14:52:26,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741862_1038 (size=20078)
2024-12-03T14:52:26,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741862_1038 (size=20078)
2024-12-03T14:52:26,961 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/c60b4ebafeb9458d836c60f1bc7a1735
2024-12-03T14:52:26,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/c60b4ebafeb9458d836c60f1bc7a1735 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/c60b4ebafeb9458d836c60f1bc7a1735
2024-12-03T14:52:26,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:26,975 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/c60b4ebafeb9458d836c60f1bc7a1735, entries=14, sequenceid=191, filesize=19.6 K
2024-12-03T14:52:26,976 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=10.51 KB/10760 for 3fa7822ea36b9984e8dc3f635eee83d8 in 28ms, sequenceid=191, compaction requested=true
2024-12-03T14:52:26,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:26,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:26,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:26,976 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:26,978 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-03T14:52:26,978 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 103780 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:26,978 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:26,978 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:26,979 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a6494aedf4114a67b04a4213933111e2, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/fc95ca3267814074a6c44dfd0c77e2e3, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/c60b4ebafeb9458d836c60f1bc7a1735] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=101.3 K
2024-12-03T14:52:26,979 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6494aedf4114a67b04a4213933111e2, keycount=61, bloomtype=ROW, size=69.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733237530593
2024-12-03T14:52:26,980 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc95ca3267814074a6c44dfd0c77e2e3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733237544895
2024-12-03T14:52:26,986 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60b4ebafeb9458d836c60f1bc7a1735, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733237546918
2024-12-03T14:52:27,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/2aa076460e4640b195a06655883d65a0 is 1080, key is row0144/info:/1733237546950/Put/seqid=0
2024-12-03T14:52:27,004 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#77 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:27,004 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/3be7ec69d1584c219d22b5e11e655e83 is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:27,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741863_1039 (size=16828)
2024-12-03T14:52:27,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741863_1039 (size=16828)
2024-12-03T14:52:27,027 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/2aa076460e4640b195a06655883d65a0
2024-12-03T14:52:27,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/2aa076460e4640b195a06655883d65a0 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/2aa076460e4640b195a06655883d65a0
2024-12-03T14:52:27,039 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/2aa076460e4640b195a06655883d65a0, entries=11, sequenceid=205, filesize=16.4 K
2024-12-03T14:52:27,040 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=6.30 KB/6456 for 3fa7822ea36b9984e8dc3f635eee83d8 in 62ms, sequenceid=205, compaction requested=false
2024-12-03T14:52:27,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:27,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741864_1040 (size=93999)
2024-12-03T14:52:27,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741864_1040 (size=93999)
2024-12-03T14:52:27,055 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/3be7ec69d1584c219d22b5e11e655e83 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3be7ec69d1584c219d22b5e11e655e83
2024-12-03T14:52:27,063 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into 3be7ec69d1584c219d22b5e11e655e83(size=91.8 K), total size for store is 108.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:27,063 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:27,063 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237546976; duration=0sec
2024-12-03T14:52:27,063 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:27,063 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:27,413 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T14:52:27,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:27,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:27,974 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:28,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:28,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:28,974 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:29,001 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:52:29,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/3011c666739a4f54b0149dae9bda0f74 is 1080, key is row0155/info:/1733237546980/Put/seqid=0
2024-12-03T14:52:29,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741865_1041 (size=12516)
2024-12-03T14:52:29,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741865_1041 (size=12516)
2024-12-03T14:52:29,024 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/3011c666739a4f54b0149dae9bda0f74
2024-12-03T14:52:29,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/3011c666739a4f54b0149dae9bda0f74 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3011c666739a4f54b0149dae9bda0f74
2024-12-03T14:52:29,035 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3011c666739a4f54b0149dae9bda0f74, entries=7, sequenceid=216, filesize=12.2 K
2024-12-03T14:52:29,036 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for 3fa7822ea36b9984e8dc3f635eee83d8 in 36ms, sequenceid=216, compaction requested=true
2024-12-03T14:52:29,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:29,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:29,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:29,037 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:29,037 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-12-03T14:52:29,038 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123343 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:29,038 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:29,038 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:29,038 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3be7ec69d1584c219d22b5e11e655e83, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/2aa076460e4640b195a06655883d65a0, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3011c666739a4f54b0149dae9bda0f74] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=120.5 K
2024-12-03T14:52:29,039 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3be7ec69d1584c219d22b5e11e655e83, keycount=82, bloomtype=ROW, size=91.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733237530593
2024-12-03T14:52:29,039 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa076460e4640b195a06655883d65a0, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733237546950
2024-12-03T14:52:29,039 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3011c666739a4f54b0149dae9bda0f74, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733237546980
2024-12-03T14:52:29,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/9f11d79b6e6543eebc495984695de24c is 1080, key is row0162/info:/1733237549002/Put/seqid=0
2024-12-03T14:52:29,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741866_1042 (size=22238)
2024-12-03T14:52:29,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741866_1042 (size=22238)
2024-12-03T14:52:29,047 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/9f11d79b6e6543eebc495984695de24c
2024-12-03T14:52:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/9f11d79b6e6543eebc495984695de24c as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/9f11d79b6e6543eebc495984695de24c
2024-12-03T14:52:29,054 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#80 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:29,054 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/7981f797691d4a27a6da84764f4a08c6 is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:29,060 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/9f11d79b6e6543eebc495984695de24c, entries=16, sequenceid=235, filesize=21.7 K
2024-12-03T14:52:29,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741867_1043 (size=113509)
2024-12-03T14:52:29,061 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=9.46 KB/9684 for 3fa7822ea36b9984e8dc3f635eee83d8 in 24ms, sequenceid=235, compaction requested=false
2024-12-03T14:52:29,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:29,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741867_1043 (size=113509)
2024-12-03T14:52:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:29,063 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-03T14:52:29,070 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/7981f797691d4a27a6da84764f4a08c6 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7981f797691d4a27a6da84764f4a08c6
2024-12-03T14:52:29,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/8d00cdadbde74f2e81b602b93f28431d is 1080, key is row0178/info:/1733237549038/Put/seqid=0
2024-12-03T14:52:29,080 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into 7981f797691d4a27a6da84764f4a08c6(size=110.8 K), total size for store is 132.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:29,080 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:29,080 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237549037; duration=0sec
2024-12-03T14:52:29,080 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:29,080 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:29,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741868_1044 (size=15750)
2024-12-03T14:52:29,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741868_1044 (size=15750)
2024-12-03T14:52:29,093 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/8d00cdadbde74f2e81b602b93f28431d
2024-12-03T14:52:29,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/8d00cdadbde74f2e81b602b93f28431d as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/8d00cdadbde74f2e81b602b93f28431d
2024-12-03T14:52:29,105 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/8d00cdadbde74f2e81b602b93f28431d, entries=10, sequenceid=248, filesize=15.4 K
2024-12-03T14:52:29,107 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=5.25 KB/5380 for 3fa7822ea36b9984e8dc3f635eee83d8 in 43ms, sequenceid=248, compaction requested=true
2024-12-03T14:52:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:29,107 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:29,108 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 151497 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:29,108 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:29,108 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:29,109 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7981f797691d4a27a6da84764f4a08c6, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/9f11d79b6e6543eebc495984695de24c, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/8d00cdadbde74f2e81b602b93f28431d] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=147.9 K
2024-12-03T14:52:29,109 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7981f797691d4a27a6da84764f4a08c6, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733237530593
2024-12-03T14:52:29,109 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f11d79b6e6543eebc495984695de24c, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733237549002
2024-12-03T14:52:29,110 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d00cdadbde74f2e81b602b93f28431d, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733237549038
2024-12-03T14:52:29,119 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#82 average throughput is 64.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:29,120 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/d4749641f3e94df480bae9071e754327 is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:29,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741869_1045 (size=141844)
2024-12-03T14:52:29,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741869_1045 (size=141844)
2024-12-03T14:52:29,129 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/d4749641f3e94df480bae9071e754327 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/d4749641f3e94df480bae9071e754327
2024-12-03T14:52:29,135 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into d4749641f3e94df480bae9071e754327(size=138.5 K), total size for store is 138.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:29,135 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:29,135 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237549107; duration=0sec
2024-12-03T14:52:29,135 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:29,135 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:29,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:29,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:29,976 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:30,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:30,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:30,976 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:31,081 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T14:52:31,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/b7ac7689a5be40f4a092871a4d6c716a is 1080, key is row0188/info:/1733237549065/Put/seqid=0
2024-12-03T14:52:31,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741870_1046 (size=12518)
2024-12-03T14:52:31,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741870_1046 (size=12518)
2024-12-03T14:52:31,090 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/b7ac7689a5be40f4a092871a4d6c716a
2024-12-03T14:52:31,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/b7ac7689a5be40f4a092871a4d6c716a as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/b7ac7689a5be40f4a092871a4d6c716a
2024-12-03T14:52:31,101 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/b7ac7689a5be40f4a092871a4d6c716a, entries=7, sequenceid=260, filesize=12.2 K
2024-12-03T14:52:31,102 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 3fa7822ea36b9984e8dc3f635eee83d8 in 21ms, sequenceid=260, compaction requested=false
2024-12-03T14:52:31,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:31,103 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-03T14:52:31,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/5df4f7d9a5114d4fb821a76717f34b00 is 1080, key is row0195/info:/1733237551082/Put/seqid=0
2024-12-03T14:52:31,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741871_1047 (size=17918)
2024-12-03T14:52:31,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741871_1047 (size=17918)
2024-12-03T14:52:31,112 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/5df4f7d9a5114d4fb821a76717f34b00
2024-12-03T14:52:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/5df4f7d9a5114d4fb821a76717f34b00 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/5df4f7d9a5114d4fb821a76717f34b00
2024-12-03T14:52:31,123 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/5df4f7d9a5114d4fb821a76717f34b00, entries=12, sequenceid=275, filesize=17.5 K
2024-12-03T14:52:31,124 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 3fa7822ea36b9984e8dc3f635eee83d8 in 21ms, sequenceid=275, compaction requested=true
2024-12-03T14:52:31,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:31,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:31,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:31,124 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:31,124 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-03T14:52:31,125 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 172280 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:31,125 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:31,125 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:31,126 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/d4749641f3e94df480bae9071e754327, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/b7ac7689a5be40f4a092871a4d6c716a, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/5df4f7d9a5114d4fb821a76717f34b00] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=168.2 K
2024-12-03T14:52:31,126 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4749641f3e94df480bae9071e754327, keycount=126, bloomtype=ROW, size=138.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733237530593
2024-12-03T14:52:31,126 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7ac7689a5be40f4a092871a4d6c716a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733237549065
2024-12-03T14:52:31,127 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df4f7d9a5114d4fb821a76717f34b00, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733237551082
2024-12-03T14:52:31,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/056af038bf4d49a08a91df61fbe95ee3 is 1080, key is row0207/info:/1733237551104/Put/seqid=0
2024-12-03T14:52:31,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741872_1048 (size=16839)
2024-12-03T14:52:31,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741872_1048 (size=16839)
2024-12-03T14:52:31,133 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/056af038bf4d49a08a91df61fbe95ee3
2024-12-03T14:52:31,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/056af038bf4d49a08a91df61fbe95ee3 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/056af038bf4d49a08a91df61fbe95ee3
2024-12-03T14:52:31,139 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#86 average throughput is 49.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:31,139 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/6c63d1ed33b04835b36dc55e6fe3abbe is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:31,143 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/056af038bf4d49a08a91df61fbe95ee3, entries=11, sequenceid=289, filesize=16.4 K
2024-12-03T14:52:31,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741873_1049 (size=162446)
2024-12-03T14:52:31,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741873_1049 (size=162446)
2024-12-03T14:52:31,144 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=7.36 KB/7532 for 3fa7822ea36b9984e8dc3f635eee83d8 in 20ms, sequenceid=289, compaction requested=false
2024-12-03T14:52:31,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:31,148 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/6c63d1ed33b04835b36dc55e6fe3abbe as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/6c63d1ed33b04835b36dc55e6fe3abbe
2024-12-03T14:52:31,153 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into 6c63d1ed33b04835b36dc55e6fe3abbe(size=158.6 K), total size for store is 175.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:31,153 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:31,153 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237551124; duration=0sec
2024-12-03T14:52:31,153 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:31,153 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:31,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:31,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:31,977 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:32,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:32,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:32,978 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:33,143 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-03T14:52:33,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/804b2b3c4faf41b8928db78dcd52e7df is 1080, key is row0218/info:/1733237551125/Put/seqid=0
2024-12-03T14:52:33,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741874_1050 (size=13602)
2024-12-03T14:52:33,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741874_1050 (size=13602)
2024-12-03T14:52:33,155 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/804b2b3c4faf41b8928db78dcd52e7df
2024-12-03T14:52:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/804b2b3c4faf41b8928db78dcd52e7df as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/804b2b3c4faf41b8928db78dcd52e7df
2024-12-03T14:52:33,165 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/804b2b3c4faf41b8928db78dcd52e7df, entries=8, sequenceid=301, filesize=13.3 K
2024-12-03T14:52:33,165 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=12.61 KB/12912 for 3fa7822ea36b9984e8dc3f635eee83d8 in 23ms, sequenceid=301, compaction requested=true
2024-12-03T14:52:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:33,166 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:33,167 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-03T14:52:33,167 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 192887 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:33,167 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:33,167 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:33,167 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/6c63d1ed33b04835b36dc55e6fe3abbe, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/056af038bf4d49a08a91df61fbe95ee3, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/804b2b3c4faf41b8928db78dcd52e7df] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=188.4 K
2024-12-03T14:52:33,168 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c63d1ed33b04835b36dc55e6fe3abbe, keycount=145, bloomtype=ROW, size=158.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733237530593
2024-12-03T14:52:33,168 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] compactions.Compactor(225): Compacting 056af038bf4d49a08a91df61fbe95ee3, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733237551104
2024-12-03T14:52:33,168 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] compactions.Compactor(225): Compacting 804b2b3c4faf41b8928db78dcd52e7df, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733237551125
2024-12-03T14:52:33,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/12356e78de514cac9b415a6127823a73 is 1080, key is row0226/info:/1733237553144/Put/seqid=0
2024-12-03T14:52:33,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741875_1051 (size=19013)
2024-12-03T14:52:33,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741875_1051 (size=19013)
2024-12-03T14:52:33,178 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/12356e78de514cac9b415a6127823a73
2024-12-03T14:52:33,181 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#89 average throughput is 56.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:33,182 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/46673298972441f395ad8e49ecea49d1 is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:33,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/12356e78de514cac9b415a6127823a73 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/12356e78de514cac9b415a6127823a73
2024-12-03T14:52:33,190 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/12356e78de514cac9b415a6127823a73, entries=13, sequenceid=317, filesize=18.6 K
2024-12-03T14:52:33,191 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=12.61 KB/12912 for 3fa7822ea36b9984e8dc3f635eee83d8 in 24ms, sequenceid=317, compaction requested=false
2024-12-03T14:52:33,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40149 {}] regionserver.HRegion(8855): Flush requested on 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:33,191 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-03T14:52:33,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/4c6d6b762f96466e9d6f6ee8c18517be is 1080, key is row0239/info:/1733237553168/Put/seqid=0
2024-12-03T14:52:33,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741876_1052 (size=183053)
2024-12-03T14:52:33,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741876_1052 (size=183053)
2024-12-03T14:52:33,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741877_1053 (size=19013)
2024-12-03T14:52:33,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741877_1053 (size=19013)
2024-12-03T14:52:33,205 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/46673298972441f395ad8e49ecea49d1 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/46673298972441f395ad8e49ecea49d1
2024-12-03T14:52:33,210 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into 46673298972441f395ad8e49ecea49d1(size=178.8 K), total size for store is 197.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:33,210 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:33,210 INFO  [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237553166; duration=0sec
2024-12-03T14:52:33,210 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:33,210 DEBUG [RS:0;a5d22df9eca2:40149-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:33,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:33,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:33,607 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/4c6d6b762f96466e9d6f6ee8c18517be
2024-12-03T14:52:33,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/4c6d6b762f96466e9d6f6ee8c18517be as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/4c6d6b762f96466e9d6f6ee8c18517be
2024-12-03T14:52:33,622 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/4c6d6b762f96466e9d6f6ee8c18517be, entries=13, sequenceid=333, filesize=18.6 K
2024-12-03T14:52:33,623 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=5.25 KB/5380 for 3fa7822ea36b9984e8dc3f635eee83d8 in 432ms, sequenceid=333, compaction requested=true
2024-12-03T14:52:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fa7822ea36b9984e8dc3f635eee83d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T14:52:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:33,623 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T14:52:33,624 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 221079 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T14:52:33,624 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1541): 3fa7822ea36b9984e8dc3f635eee83d8/info is initiating minor compaction (all files)
2024-12-03T14:52:33,624 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fa7822ea36b9984e8dc3f635eee83d8/info in TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:33,624 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/46673298972441f395ad8e49ecea49d1, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/12356e78de514cac9b415a6127823a73, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/4c6d6b762f96466e9d6f6ee8c18517be] into tmpdir=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp, totalSize=215.9 K
2024-12-03T14:52:33,624 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46673298972441f395ad8e49ecea49d1, keycount=164, bloomtype=ROW, size=178.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733237530593
2024-12-03T14:52:33,625 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12356e78de514cac9b415a6127823a73, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733237553144
2024-12-03T14:52:33,625 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c6d6b762f96466e9d6f6ee8c18517be, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733237553168
2024-12-03T14:52:33,635 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fa7822ea36b9984e8dc3f635eee83d8#info#compaction#91 average throughput is 64.99 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T14:52:33,635 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/5a5e5643d64f45a5ac50fe346ecad44b is 1080, key is row0062/info:/1733237530593/Put/seqid=0
2024-12-03T14:52:33,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741878_1054 (size=211286)
2024-12-03T14:52:33,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741878_1054 (size=211286)
2024-12-03T14:52:33,643 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/5a5e5643d64f45a5ac50fe346ecad44b as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/5a5e5643d64f45a5ac50fe346ecad44b
2024-12-03T14:52:33,648 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fa7822ea36b9984e8dc3f635eee83d8/info of 3fa7822ea36b9984e8dc3f635eee83d8 into 5a5e5643d64f45a5ac50fe346ecad44b(size=206.3 K), total size for store is 206.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T14:52:33,648 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:33,648 INFO  [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., storeName=3fa7822ea36b9984e8dc3f635eee83d8/info, priority=13, startTime=1733237553623; duration=0sec
2024-12-03T14:52:33,648 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T14:52:33,648 DEBUG [RS:0;a5d22df9eca2:40149-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fa7822ea36b9984e8dc3f635eee83d8:info
2024-12-03T14:52:33,979 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:34,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:34,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:34,979 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:35,202 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-03T14:52:35,202 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C40149%2C1733237517474.1733237555202
2024-12-03T14:52:35,210 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,210 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,210 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,210 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,211 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,211 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237517897 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237555202
2024-12-03T14:52:35,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741833_1009 (size=317837)
2024-12-03T14:52:35,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35851:35851),(127.0.0.1/127.0.0.1:45073:45073)]
2024-12-03T14:52:35,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237517897 is not closed yet, will try archiving it next time
2024-12-03T14:52:35,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741833_1009 (size=317837)
2024-12-03T14:52:35,221 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 85fe1f7ea138f1a0fc13ce67fd291276:

2024-12-03T14:52:35,221 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 3fa7822ea36b9984e8dc3f635eee83d8 1/1 column families, dataSize=5.25 KB heapSize=5.88 KB
2024-12-03T14:52:35,227 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/90d6072e448d480e9c77a4122e284194 is 1080, key is row0252/info:/1733237553192/Put/seqid=0
2024-12-03T14:52:35,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741880_1056 (size=10357)
2024-12-03T14:52:35,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741880_1056 (size=10357)
2024-12-03T14:52:35,246 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.25 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/90d6072e448d480e9c77a4122e284194
2024-12-03T14:52:35,252 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/.tmp/info/90d6072e448d480e9c77a4122e284194 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/90d6072e448d480e9c77a4122e284194
2024-12-03T14:52:35,259 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/90d6072e448d480e9c77a4122e284194, entries=5, sequenceid=343, filesize=10.1 K
2024-12-03T14:52:35,261 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 3fa7822ea36b9984e8dc3f635eee83d8 in 40ms, sequenceid=343, compaction requested=false
2024-12-03T14:52:35,261 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3fa7822ea36b9984e8dc3f635eee83d8:

2024-12-03T14:52:35,261 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-12-03T14:52:35,270 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/info/6200969d0fdf407ab667c90df1605401 is 186, key is TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276./info:regioninfo/1733237533399/Put/seqid=0
2024-12-03T14:52:35,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741881_1057 (size=6153)
2024-12-03T14:52:35,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741881_1057 (size=6153)
2024-12-03T14:52:35,309 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/info/6200969d0fdf407ab667c90df1605401
2024-12-03T14:52:35,314 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/.tmp/info/6200969d0fdf407ab667c90df1605401 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/info/6200969d0fdf407ab667c90df1605401
2024-12-03T14:52:35,319 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/info/6200969d0fdf407ab667c90df1605401, entries=5, sequenceid=21, filesize=6.0 K
2024-12-03T14:52:35,320 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=21, compaction requested=false
2024-12-03T14:52:35,320 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-03T14:52:35,321 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C40149%2C1733237517474.1733237555321
2024-12-03T14:52:35,334 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,334 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,334 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,334 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,334 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,334 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237555202 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237555321
2024-12-03T14:52:35,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741879_1055 (size=731)
2024-12-03T14:52:35,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741879_1055 (size=731)
2024-12-03T14:52:35,340 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237517897 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/oldWALs/a5d22df9eca2%2C40149%2C1733237517474.1733237517897
2024-12-03T14:52:35,342 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/WALs/a5d22df9eca2,40149,1733237517474/a5d22df9eca2%2C40149%2C1733237517474.1733237555202 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/oldWALs/a5d22df9eca2%2C40149%2C1733237517474.1733237555202
2024-12-03T14:52:35,345 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45073:45073),(127.0.0.1/127.0.0.1:35851:35851)]
2024-12-03T14:52:35,346 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-03T14:52:35,346 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-03T14:52:35,346 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:52:35,346 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:52:35,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:35,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:35,347 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-03T14:52:35,347 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T14:52:35,347 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1555767589, stopped=false
2024-12-03T14:52:35,347 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,34805,1733237517434
2024-12-03T14:52:35,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:52:35,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:52:35,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:35,348 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:52:35,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:35,349 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:52:35,349 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:52:35,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:35,349 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,40149,1733237517474' *****
2024-12-03T14:52:35,349 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:52:35,349 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:52:35,350 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:52:35,350 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(3091): Received CLOSE for 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(3091): Received CLOSE for 3fa7822ea36b9984e8dc3f635eee83d8
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:52:35,350 INFO  [RS:0;a5d22df9eca2:40149 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:40149.
2024-12-03T14:52:35,350 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 85fe1f7ea138f1a0fc13ce67fd291276, disabling compactions & flushes
2024-12-03T14:52:35,350 DEBUG [RS:0;a5d22df9eca2:40149 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:52:35,350 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:35,351 DEBUG [RS:0;a5d22df9eca2:40149 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:35,351 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:35,351 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276. after waiting 0 ms
2024-12-03T14:52:35,351 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:35,351 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:52:35,351 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:52:35,351 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:52:35,351 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-03T14:52:35,353 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93->hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb-bottom] to archive
2024-12-03T14:52:35,355 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T14:52:35,355 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-03T14:52:35,355 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1325): Online Regions={85fe1f7ea138f1a0fc13ce67fd291276=TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276., 3fa7822ea36b9984e8dc3f635eee83d8=TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8., 1588230740=hbase:meta,,1.1588230740}
2024-12-03T14:52:35,355 DEBUG [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3fa7822ea36b9984e8dc3f635eee83d8, 85fe1f7ea138f1a0fc13ce67fd291276
2024-12-03T14:52:35,356 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:35,357 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a5d22df9eca2:34805 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-03T14:52:35,357 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-03T14:52:35,361 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:52:35,361 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:52:35,361 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:52:35,361 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:52:35,361 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:52:35,378 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/85fe1f7ea138f1a0fc13ce67fd291276/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-12-03T14:52:35,379 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-03T14:52:35,379 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:35,379 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 85fe1f7ea138f1a0fc13ce67fd291276:
Waiting for close lock at 1733237555350Running coprocessor pre-close hooks at 1733237555350Disabling compacts and flushes for region at 1733237555350Disabling writes for close at 1733237555351 (+1 ms)Writing region close event to WAL at 1733237555365 (+14 ms)Running coprocessor post-close hooks at 1733237555379 (+14 ms)Closed at 1733237555379
2024-12-03T14:52:35,379 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733237532693.85fe1f7ea138f1a0fc13ce67fd291276.
2024-12-03T14:52:35,379 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:52:35,380 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:52:35,380 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3fa7822ea36b9984e8dc3f635eee83d8, disabling compactions & flushes
2024-12-03T14:52:35,380 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237555361Running coprocessor pre-close hooks at 1733237555361Disabling compacts and flushes for region at 1733237555361Disabling writes for close at 1733237555361Writing region close event to WAL at 1733237555369 (+8 ms)Running coprocessor post-close hooks at 1733237555379 (+10 ms)Closed at 1733237555380 (+1 ms)
2024-12-03T14:52:35,380 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:35,380 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:35,380 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T14:52:35,380 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8. after waiting 0 ms
2024-12-03T14:52:35,380 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:35,380 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93->hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/fbc3069cf266c503faf2cea6b61c8f93/info/a497216126c14c088fc6d929804b1bfb-top, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7f4a81bb51d3417e81679a89815ce1d5, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/f770c62cc32f46be93590b3a562b752a, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a6494aedf4114a67b04a4213933111e2, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/af16678f8b8e4f8da42821b4f8cd340d, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/fc95ca3267814074a6c44dfd0c77e2e3, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3be7ec69d1584c219d22b5e11e655e83, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/c60b4ebafeb9458d836c60f1bc7a1735, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/2aa076460e4640b195a06655883d65a0, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7981f797691d4a27a6da84764f4a08c6, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3011c666739a4f54b0149dae9bda0f74, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/9f11d79b6e6543eebc495984695de24c, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/d4749641f3e94df480bae9071e754327, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/8d00cdadbde74f2e81b602b93f28431d, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/b7ac7689a5be40f4a092871a4d6c716a, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/6c63d1ed33b04835b36dc55e6fe3abbe, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/5df4f7d9a5114d4fb821a76717f34b00, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/056af038bf4d49a08a91df61fbe95ee3, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/46673298972441f395ad8e49ecea49d1, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/804b2b3c4faf41b8928db78dcd52e7df, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/12356e78de514cac9b415a6127823a73, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/4c6d6b762f96466e9d6f6ee8c18517be] to archive
2024-12-03T14:52:35,382 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T14:52:35,384 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a497216126c14c088fc6d929804b1bfb.fbc3069cf266c503faf2cea6b61c8f93
2024-12-03T14:52:35,385 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-e4c26fec8329454ca7a76e30b5219545
2024-12-03T14:52:35,387 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7f4a81bb51d3417e81679a89815ce1d5 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7f4a81bb51d3417e81679a89815ce1d5
2024-12-03T14:52:35,389 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/TestLogRolling-testLogRolling=fbc3069cf266c503faf2cea6b61c8f93-23e44889b2b948218eb99e4d42a88691
2024-12-03T14:52:35,391 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/f770c62cc32f46be93590b3a562b752a to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/f770c62cc32f46be93590b3a562b752a
2024-12-03T14:52:35,393 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a6494aedf4114a67b04a4213933111e2 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/a6494aedf4114a67b04a4213933111e2
2024-12-03T14:52:35,395 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/af16678f8b8e4f8da42821b4f8cd340d to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/af16678f8b8e4f8da42821b4f8cd340d
2024-12-03T14:52:35,397 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/fc95ca3267814074a6c44dfd0c77e2e3 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/fc95ca3267814074a6c44dfd0c77e2e3
2024-12-03T14:52:35,398 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3be7ec69d1584c219d22b5e11e655e83 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3be7ec69d1584c219d22b5e11e655e83
2024-12-03T14:52:35,400 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/c60b4ebafeb9458d836c60f1bc7a1735 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/c60b4ebafeb9458d836c60f1bc7a1735
2024-12-03T14:52:35,402 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/2aa076460e4640b195a06655883d65a0 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/2aa076460e4640b195a06655883d65a0
2024-12-03T14:52:35,404 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7981f797691d4a27a6da84764f4a08c6 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/7981f797691d4a27a6da84764f4a08c6
2024-12-03T14:52:35,406 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3011c666739a4f54b0149dae9bda0f74 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/3011c666739a4f54b0149dae9bda0f74
2024-12-03T14:52:35,409 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/9f11d79b6e6543eebc495984695de24c to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/9f11d79b6e6543eebc495984695de24c
2024-12-03T14:52:35,411 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/d4749641f3e94df480bae9071e754327 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/d4749641f3e94df480bae9071e754327
2024-12-03T14:52:35,412 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/8d00cdadbde74f2e81b602b93f28431d to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/8d00cdadbde74f2e81b602b93f28431d
2024-12-03T14:52:35,413 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/b7ac7689a5be40f4a092871a4d6c716a to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/b7ac7689a5be40f4a092871a4d6c716a
2024-12-03T14:52:35,415 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/6c63d1ed33b04835b36dc55e6fe3abbe to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/6c63d1ed33b04835b36dc55e6fe3abbe
2024-12-03T14:52:35,416 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/5df4f7d9a5114d4fb821a76717f34b00 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/5df4f7d9a5114d4fb821a76717f34b00
2024-12-03T14:52:35,422 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/056af038bf4d49a08a91df61fbe95ee3 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/056af038bf4d49a08a91df61fbe95ee3
2024-12-03T14:52:35,423 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/46673298972441f395ad8e49ecea49d1 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/46673298972441f395ad8e49ecea49d1
2024-12-03T14:52:35,424 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/804b2b3c4faf41b8928db78dcd52e7df to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/804b2b3c4faf41b8928db78dcd52e7df
2024-12-03T14:52:35,426 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/12356e78de514cac9b415a6127823a73 to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/12356e78de514cac9b415a6127823a73
2024-12-03T14:52:35,427 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/4c6d6b762f96466e9d6f6ee8c18517be to hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/archive/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/info/4c6d6b762f96466e9d6f6ee8c18517be
2024-12-03T14:52:35,427 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [7f4a81bb51d3417e81679a89815ce1d5=42984, f770c62cc32f46be93590b3a562b752a=12516, a6494aedf4114a67b04a4213933111e2=71186, af16678f8b8e4f8da42821b4f8cd340d=25472, fc95ca3267814074a6c44dfd0c77e2e3=12516, 3be7ec69d1584c219d22b5e11e655e83=93999, c60b4ebafeb9458d836c60f1bc7a1735=20078, 2aa076460e4640b195a06655883d65a0=16828, 7981f797691d4a27a6da84764f4a08c6=113509, 3011c666739a4f54b0149dae9bda0f74=12516, 9f11d79b6e6543eebc495984695de24c=22238, d4749641f3e94df480bae9071e754327=141844, 8d00cdadbde74f2e81b602b93f28431d=15750, b7ac7689a5be40f4a092871a4d6c716a=12518, 6c63d1ed33b04835b36dc55e6fe3abbe=162446, 5df4f7d9a5114d4fb821a76717f34b00=17918, 056af038bf4d49a08a91df61fbe95ee3=16839, 46673298972441f395ad8e49ecea49d1=183053, 804b2b3c4faf41b8928db78dcd52e7df=13602, 12356e78de514cac9b415a6127823a73=19013, 4c6d6b762f96466e9d6f6ee8c18517be=19013]
2024-12-03T14:52:35,433 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/data/default/TestLogRolling-testLogRolling/3fa7822ea36b9984e8dc3f635eee83d8/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=130
2024-12-03T14:52:35,433 INFO  [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:35,434 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3fa7822ea36b9984e8dc3f635eee83d8:
Waiting for close lock at 1733237555380Running coprocessor pre-close hooks at 1733237555380Disabling compacts and flushes for region at 1733237555380Disabling writes for close at 1733237555380Writing region close event to WAL at 1733237555428 (+48 ms)Running coprocessor post-close hooks at 1733237555433 (+5 ms)Closed at 1733237555433
2024-12-03T14:52:35,434 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733237532693.3fa7822ea36b9984e8dc3f635eee83d8.
2024-12-03T14:52:35,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:35,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:35,555 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,40149,1733237517474; all regions closed.
2024-12-03T14:52:35,556 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,556 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,556 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,556 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,556 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741834_1010 (size=8107)
2024-12-03T14:52:35,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741834_1010 (size=8107)
2024-12-03T14:52:35,563 DEBUG [RS:0;a5d22df9eca2:40149 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/oldWALs
2024-12-03T14:52:35,563 INFO  [RS:0;a5d22df9eca2:40149 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C40149%2C1733237517474.meta:.meta(num 1733237518312)
2024-12-03T14:52:35,564 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,564 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,564 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,564 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,564 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741882_1058 (size=780)
2024-12-03T14:52:35,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741882_1058 (size=780)
2024-12-03T14:52:35,570 DEBUG [RS:0;a5d22df9eca2:40149 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/oldWALs
2024-12-03T14:52:35,570 INFO  [RS:0;a5d22df9eca2:40149 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C40149%2C1733237517474:(num 1733237555321)
2024-12-03T14:52:35,570 DEBUG [RS:0;a5d22df9eca2:40149 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:35,570 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:52:35,571 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:52:35,571 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-03T14:52:35,571 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:52:35,572 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:52:35,572 INFO  [RS:0;a5d22df9eca2:40149 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40149
2024-12-03T14:52:35,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:52:35,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,40149,1733237517474
2024-12-03T14:52:35,573 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:52:35,574 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,40149,1733237517474]
2024-12-03T14:52:35,575 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,40149,1733237517474 already deleted, retry=false
2024-12-03T14:52:35,575 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,40149,1733237517474 expired; onlineServers=0
2024-12-03T14:52:35,575 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,34805,1733237517434' *****
2024-12-03T14:52:35,575 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T14:52:35,575 INFO  [M:0;a5d22df9eca2:34805 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:52:35,575 INFO  [M:0;a5d22df9eca2:34805 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:52:35,575 DEBUG [M:0;a5d22df9eca2:34805 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T14:52:35,575 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T14:52:35,575 DEBUG [M:0;a5d22df9eca2:34805 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T14:52:35,575 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237517675 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237517675,5,FailOnTimeoutGroup]
2024-12-03T14:52:35,575 INFO  [M:0;a5d22df9eca2:34805 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-03T14:52:35,575 INFO  [M:0;a5d22df9eca2:34805 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:52:35,575 DEBUG [M:0;a5d22df9eca2:34805 {}] master.HMaster(1795): Stopping service threads
2024-12-03T14:52:35,575 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237517675 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237517675,5,FailOnTimeoutGroup]
2024-12-03T14:52:35,575 INFO  [M:0;a5d22df9eca2:34805 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T14:52:35,575 INFO  [M:0;a5d22df9eca2:34805 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:52:35,575 INFO  [M:0;a5d22df9eca2:34805 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T14:52:35,575 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T14:52:35,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T14:52:35,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:35,578 DEBUG [M:0;a5d22df9eca2:34805 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-12-03T14:52:35,579 DEBUG [M:0;a5d22df9eca2:34805 {}] master.ActiveMasterManager(353): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-12-03T14:52:35,579 INFO  [M:0;a5d22df9eca2:34805 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/.lastflushedseqids
2024-12-03T14:52:35,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741883_1059 (size=228)
2024-12-03T14:52:35,616 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741883_1059 (size=228)
2024-12-03T14:52:35,618 INFO  [M:0;a5d22df9eca2:34805 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-03T14:52:35,618 INFO  [M:0;a5d22df9eca2:34805 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T14:52:35,618 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:52:35,618 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:35,618 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:35,618 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:52:35,618 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:35,618 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB
2024-12-03T14:52:35,640 DEBUG [M:0;a5d22df9eca2:34805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd9d8c09e7f549e9816503567d3f31d7 is 82, key is hbase:meta,,1/info:regioninfo/1733237518341/Put/seqid=0
2024-12-03T14:52:35,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741884_1060 (size=5672)
2024-12-03T14:52:35,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741884_1060 (size=5672)
2024-12-03T14:52:35,645 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd9d8c09e7f549e9816503567d3f31d7
2024-12-03T14:52:35,665 DEBUG [M:0;a5d22df9eca2:34805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b489a58c69394f1eb1beddf08c538256 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733237518799/Put/seqid=0
2024-12-03T14:52:35,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741885_1061 (size=7090)
2024-12-03T14:52:35,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741885_1061 (size=7090)
2024-12-03T14:52:35,671 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b489a58c69394f1eb1beddf08c538256
2024-12-03T14:52:35,675 INFO  [RS:0;a5d22df9eca2:40149 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:52:35,675 INFO  [RS:0;a5d22df9eca2:40149 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,40149,1733237517474; zookeeper connection closed.
2024-12-03T14:52:35,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:35,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40149-0x100a09108d60001, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:35,675 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a43e0f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a43e0f
2024-12-03T14:52:35,675 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T14:52:35,676 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b489a58c69394f1eb1beddf08c538256
2024-12-03T14:52:35,696 DEBUG [M:0;a5d22df9eca2:34805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ad828aecdd24178991a2dbe3c955439 is 69, key is a5d22df9eca2,40149,1733237517474/rs:state/1733237517719/Put/seqid=0
2024-12-03T14:52:35,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741886_1062 (size=5156)
2024-12-03T14:52:35,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741886_1062 (size=5156)
2024-12-03T14:52:35,713 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ad828aecdd24178991a2dbe3c955439
2024-12-03T14:52:35,738 DEBUG [M:0;a5d22df9eca2:34805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e9cf498d98374bf891ea50fa23af1810 is 52, key is load_balancer_on/state:d/1733237518401/Put/seqid=0
2024-12-03T14:52:35,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741887_1063 (size=5056)
2024-12-03T14:52:35,747 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741887_1063 (size=5056)
2024-12-03T14:52:35,748 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e9cf498d98374bf891ea50fa23af1810
2024-12-03T14:52:35,753 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd9d8c09e7f549e9816503567d3f31d7 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cd9d8c09e7f549e9816503567d3f31d7
2024-12-03T14:52:35,754 INFO  [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:52:35,758 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cd9d8c09e7f549e9816503567d3f31d7, entries=8, sequenceid=125, filesize=5.5 K
2024-12-03T14:52:35,759 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b489a58c69394f1eb1beddf08c538256 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b489a58c69394f1eb1beddf08c538256
2024-12-03T14:52:35,764 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b489a58c69394f1eb1beddf08c538256
2024-12-03T14:52:35,764 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b489a58c69394f1eb1beddf08c538256, entries=13, sequenceid=125, filesize=6.9 K
2024-12-03T14:52:35,765 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ad828aecdd24178991a2dbe3c955439 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1ad828aecdd24178991a2dbe3c955439
2024-12-03T14:52:35,769 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1ad828aecdd24178991a2dbe3c955439, entries=1, sequenceid=125, filesize=5.0 K
2024-12-03T14:52:35,770 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e9cf498d98374bf891ea50fa23af1810 as hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e9cf498d98374bf891ea50fa23af1810
2024-12-03T14:52:35,776 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38757/user/jenkins/test-data/0b82cdcc-865e-60fa-2fc0-9b64fb77b99e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e9cf498d98374bf891ea50fa23af1810, entries=1, sequenceid=125, filesize=4.9 K
2024-12-03T14:52:35,777 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 159ms, sequenceid=125, compaction requested=false
2024-12-03T14:52:35,778 INFO  [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:35,779 DEBUG [M:0;a5d22df9eca2:34805 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237555618Disabling compacts and flushes for region at 1733237555618Disabling writes for close at 1733237555618Obtaining lock to block concurrent updates at 1733237555618Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237555618Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1733237555619 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237555620 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237555620Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237555640 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237555640Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237555650 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237555664 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237555664Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237555677 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237555695 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237555695Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733237555718 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733237555738 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733237555738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e67eb59: reopening flushed file at 1733237555752 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68ff43a4: reopening flushed file at 1733237555758 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@417e59bc: reopening flushed file at 1733237555764 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3771f26a: reopening flushed file at 1733237555769 (+5 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 159ms, sequenceid=125, compaction requested=false at 1733237555777 (+8 ms)Writing region close event to WAL at 1733237555778 (+1 ms)Closed at 1733237555778
2024-12-03T14:52:35,780 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,781 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,781 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,781 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,781 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:35,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43195 is added to blk_1073741830_1006 (size=61332)
2024-12-03T14:52:35,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741830_1006 (size=61332)
2024-12-03T14:52:35,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:36,184 INFO  [M:0;a5d22df9eca2:34805 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-03T14:52:36,184 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:52:36,184 INFO  [M:0;a5d22df9eca2:34805 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34805
2024-12-03T14:52:36,184 INFO  [M:0;a5d22df9eca2:34805 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:52:36,288 INFO  [M:0;a5d22df9eca2:34805 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:52:36,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:36,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34805-0x100a09108d60000, quorum=127.0.0.1:62151, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:36,301 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@551592b1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:52:36,302 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7193a060{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:52:36,302 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:52:36,302 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@277e18bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:52:36,302 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26881465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir/,STOPPED}
2024-12-03T14:52:36,304 WARN  [BP-1721694701-172.17.0.2-1733237516819 heartbeating to localhost/127.0.0.1:38757 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:52:36,304 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:52:36,304 WARN  [BP-1721694701-172.17.0.2-1733237516819 heartbeating to localhost/127.0.0.1:38757 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1721694701-172.17.0.2-1733237516819 (Datanode Uuid fd8a4e44-b149-4938-856d-1a7328dc9553) service to localhost/127.0.0.1:38757
2024-12-03T14:52:36,304 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:52:36,304 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data3/current/BP-1721694701-172.17.0.2-1733237516819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:36,305 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data4/current/BP-1721694701-172.17.0.2-1733237516819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:36,305 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:52:36,322 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4181d37d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:52:36,322 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c10c155{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:52:36,322 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:52:36,322 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47b62c1f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:52:36,323 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36eaef54{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir/,STOPPED}
2024-12-03T14:52:36,324 WARN  [BP-1721694701-172.17.0.2-1733237516819 heartbeating to localhost/127.0.0.1:38757 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:52:36,324 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:52:36,324 WARN  [BP-1721694701-172.17.0.2-1733237516819 heartbeating to localhost/127.0.0.1:38757 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1721694701-172.17.0.2-1733237516819 (Datanode Uuid 948de6bc-b476-47b3-bdef-9e82cbcc4980) service to localhost/127.0.0.1:38757
2024-12-03T14:52:36,324 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:52:36,325 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data1/current/BP-1721694701-172.17.0.2-1733237516819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:36,325 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/cluster_b8c2d694-e209-b02d-d85c-d4de000a930d/data/data2/current/BP-1721694701-172.17.0.2-1733237516819 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:36,325 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:52:36,332 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@8e4b628{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:52:36,333 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7e466bdb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:52:36,333 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:52:36,333 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75ee0a96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:52:36,333 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c28d9ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir/,STOPPED}
2024-12-03T14:52:36,342 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-03T14:52:36,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-03T14:52:36,383 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=231 (was 207)
Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38757
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38757
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38757 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:38757
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38757 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38757
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38757
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38757 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 137) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4660 (was 4758)
2024-12-03T14:52:36,391 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=231, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=4660
2024-12-03T14:52:36,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.log.dir so I do NOT create it in target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1f61fc5-d555-42b7-9406-d9bd76a1dc58/hadoop.tmp.dir so I do NOT create it in target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84, deleteOnExit=true
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/test.cache.data in system properties and HBase conf
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.log.dir in system properties and HBase conf
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T14:52:36,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-03T14:52:36,392 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:52:36,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T14:52:36,394 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/nfs.dump.dir in system properties and HBase conf
2024-12-03T14:52:36,394 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/java.io.tmpdir in system properties and HBase conf
2024-12-03T14:52:36,394 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T14:52:36,394 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T14:52:36,394 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T14:52:36,411 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:52:36,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:36,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:36,467 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:52:36,474 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:52:36,475 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:52:36,476 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:52:36,476 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:52:36,477 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:52:36,481 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fee3652{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:52:36,481 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@412c5a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:52:36,572 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4bedee20{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/java.io.tmpdir/jetty-localhost-37809-hadoop-hdfs-3_4_1-tests_jar-_-any-3787610321479305891/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:52:36,573 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@27862096{HTTP/1.1, (http/1.1)}{localhost:37809}
2024-12-03T14:52:36,573 INFO  [Time-limited test {}] server.Server(415): Started @279879ms
2024-12-03T14:52:36,583 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T14:52:36,618 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:52:36,621 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:52:36,622 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:52:36,622 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:52:36,622 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:52:36,623 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5be97557{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:52:36,623 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10c1adfc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:52:36,712 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@150dab73{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/java.io.tmpdir/jetty-localhost-45799-hadoop-hdfs-3_4_1-tests_jar-_-any-2980533139770250391/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:52:36,712 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6486a7e6{HTTP/1.1, (http/1.1)}{localhost:45799}
2024-12-03T14:52:36,712 INFO  [Time-limited test {}] server.Server(415): Started @280019ms
2024-12-03T14:52:36,713 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:52:36,739 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T14:52:36,741 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T14:52:36,742 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T14:52:36,742 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T14:52:36,742 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T14:52:36,743 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b825638{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.log.dir/,AVAILABLE}
2024-12-03T14:52:36,744 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1926aa54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T14:52:36,767 WARN  [Thread-2493 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data1/current/BP-1438476385-172.17.0.2-1733237556415/current, will proceed with Du for space computation calculation, 
2024-12-03T14:52:36,767 WARN  [Thread-2494 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data2/current/BP-1438476385-172.17.0.2-1733237556415/current, will proceed with Du for space computation calculation, 
2024-12-03T14:52:36,789 WARN  [Thread-2472 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:52:36,792 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c4a6da9a2198621 with lease ID 0xb2ac1b561aec160b: Processing first storage report for DS-644bfe12-5ae7-472b-aed5-bd5e40be2520 from datanode DatanodeRegistration(127.0.0.1:37441, datanodeUuid=3c2adf39-8364-4393-aadb-950429fbcd1b, infoPort=42863, infoSecurePort=0, ipcPort=34937, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415)
2024-12-03T14:52:36,792 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c4a6da9a2198621 with lease ID 0xb2ac1b561aec160b: from storage DS-644bfe12-5ae7-472b-aed5-bd5e40be2520 node DatanodeRegistration(127.0.0.1:37441, datanodeUuid=3c2adf39-8364-4393-aadb-950429fbcd1b, infoPort=42863, infoSecurePort=0, ipcPort=34937, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:52:36,792 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c4a6da9a2198621 with lease ID 0xb2ac1b561aec160b: Processing first storage report for DS-74adcf46-4834-46cd-8a57-eb6341540e83 from datanode DatanodeRegistration(127.0.0.1:37441, datanodeUuid=3c2adf39-8364-4393-aadb-950429fbcd1b, infoPort=42863, infoSecurePort=0, ipcPort=34937, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415)
2024-12-03T14:52:36,792 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c4a6da9a2198621 with lease ID 0xb2ac1b561aec160b: from storage DS-74adcf46-4834-46cd-8a57-eb6341540e83 node DatanodeRegistration(127.0.0.1:37441, datanodeUuid=3c2adf39-8364-4393-aadb-950429fbcd1b, infoPort=42863, infoSecurePort=0, ipcPort=34937, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:52:36,839 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@380ffe40{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/java.io.tmpdir/jetty-localhost-46705-hadoop-hdfs-3_4_1-tests_jar-_-any-15168811701301490532/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:52:36,839 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e35321a{HTTP/1.1, (http/1.1)}{localhost:46705}
2024-12-03T14:52:36,839 INFO  [Time-limited test {}] server.Server(415): Started @280146ms
2024-12-03T14:52:36,840 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T14:52:36,891 WARN  [Thread-2519 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data3/current/BP-1438476385-172.17.0.2-1733237556415/current, will proceed with Du for space computation calculation, 
2024-12-03T14:52:36,891 WARN  [Thread-2520 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data4/current/BP-1438476385-172.17.0.2-1733237556415/current, will proceed with Du for space computation calculation, 
2024-12-03T14:52:36,913 WARN  [Thread-2508 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T14:52:36,915 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x430097403c1517c with lease ID 0xb2ac1b561aec160c: Processing first storage report for DS-990be0b0-da00-4a1e-ba4c-f3d91d0040b2 from datanode DatanodeRegistration(127.0.0.1:46219, datanodeUuid=daa912b9-4b4f-456d-86ca-5703ba74c39f, infoPort=39807, infoSecurePort=0, ipcPort=44525, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415)
2024-12-03T14:52:36,915 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x430097403c1517c with lease ID 0xb2ac1b561aec160c: from storage DS-990be0b0-da00-4a1e-ba4c-f3d91d0040b2 node DatanodeRegistration(127.0.0.1:46219, datanodeUuid=daa912b9-4b4f-456d-86ca-5703ba74c39f, infoPort=39807, infoSecurePort=0, ipcPort=44525, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:52:36,915 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x430097403c1517c with lease ID 0xb2ac1b561aec160c: Processing first storage report for DS-a2917f6f-4f39-400f-958e-5df98f4f411d from datanode DatanodeRegistration(127.0.0.1:46219, datanodeUuid=daa912b9-4b4f-456d-86ca-5703ba74c39f, infoPort=39807, infoSecurePort=0, ipcPort=44525, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415)
2024-12-03T14:52:36,915 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x430097403c1517c with lease ID 0xb2ac1b561aec160c: from storage DS-a2917f6f-4f39-400f-958e-5df98f4f411d node DatanodeRegistration(127.0.0.1:46219, datanodeUuid=daa912b9-4b4f-456d-86ca-5703ba74c39f, infoPort=39807, infoSecurePort=0, ipcPort=44525, storageInfo=lv=-57;cid=testClusterID;nsid=1737568972;c=1733237556415), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T14:52:36,958 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f
2024-12-03T14:52:36,961 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/zookeeper_0, clientPort=63628, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T14:52:36,961 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63628
2024-12-03T14:52:36,962 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:36,963 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:36,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:52:36,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741825_1001 (size=7)
2024-12-03T14:52:36,971 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4 with version=8
2024-12-03T14:52:36,971 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43675/user/jenkins/test-data/b7ace9a9-4b6c-4cdd-d261-6ac4b010adea/hbase-staging
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-03T14:52:36,973 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:52:36,974 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46245
2024-12-03T14:52:36,975 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46245 connecting to ZooKeeper ensemble=127.0.0.1:63628
2024-12-03T14:52:36,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462450x0, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:52:36,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46245-0x100a091a34f0000 connected
2024-12-03T14:52:36,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:36,993 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:36,994 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:36,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:52:36,995 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4, hbase.cluster.distributed=false
2024-12-03T14:52:36,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:52:36,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46245
2024-12-03T14:52:36,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46245
2024-12-03T14:52:36,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46245
2024-12-03T14:52:37,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46245
2024-12-03T14:52:37,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46245
2024-12-03T14:52:37,013 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45
2024-12-03T14:52:37,013 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:52:37,013 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T14:52:37,013 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T14:52:37,013 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T14:52:37,013 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T14:52:37,013 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T14:52:37,014 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T14:52:37,014 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45679
2024-12-03T14:52:37,015 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45679 connecting to ZooKeeper ensemble=127.0.0.1:63628
2024-12-03T14:52:37,015 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:37,017 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:37,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456790x0, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T14:52:37,021 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456790x0, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:52:37,021 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45679-0x100a091a34f0001 connected
2024-12-03T14:52:37,021 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T14:52:37,021 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T14:52:37,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T14:52:37,023 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T14:52:37,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45679
2024-12-03T14:52:37,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45679
2024-12-03T14:52:37,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45679
2024-12-03T14:52:37,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45679
2024-12-03T14:52:37,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45679
2024-12-03T14:52:37,038 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:46245
2024-12-03T14:52:37,039 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:52:37,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:52:37,040 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T14:52:37,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,041 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T14:52:37,041 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,46245,1733237556973 from backup master directory
2024-12-03T14:52:37,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:52:37,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T14:52:37,042 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:52:37,042 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,046 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/hbase.id] with ID: d58174b8-6648-4218-83a0-ad29b19b1be3
2024-12-03T14:52:37,046 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/.tmp/hbase.id
2024-12-03T14:52:37,055 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:52:37,055 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741826_1002 (size=42)
2024-12-03T14:52:37,056 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/.tmp/hbase.id]:[hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/hbase.id]
2024-12-03T14:52:37,065 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:37,065 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-03T14:52:37,066 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-03T14:52:37,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:52:37,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741827_1003 (size=196)
2024-12-03T14:52:37,074 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T14:52:37,074 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T14:52:37,075 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:52:37,083 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:52:37,083 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741828_1004 (size=1189)
2024-12-03T14:52:37,084 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store
2024-12-03T14:52:37,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:52:37,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741829_1005 (size=34)
2024-12-03T14:52:37,092 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:52:37,092 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:52:37,092 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:37,092 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:37,092 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:52:37,092 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:37,092 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:37,092 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237557092Disabling compacts and flushes for region at 1733237557092Disabling writes for close at 1733237557092Writing region close event to WAL at 1733237557092Closed at 1733237557092
2024-12-03T14:52:37,092 WARN  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/.initializing
2024-12-03T14:52:37,093 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/WALs/a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,095 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C46245%2C1733237556973, suffix=, logDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/WALs/a5d22df9eca2,46245,1733237556973, archiveDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/oldWALs, maxLogs=10
2024-12-03T14:52:37,095 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C46245%2C1733237556973.1733237557095
2024-12-03T14:52:37,099 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/WALs/a5d22df9eca2,46245,1733237556973/a5d22df9eca2%2C46245%2C1733237556973.1733237557095
2024-12-03T14:52:37,100 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39807:39807),(127.0.0.1/127.0.0.1:42863:42863)]
2024-12-03T14:52:37,109 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:52:37,109 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:52:37,109 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,109 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:52:37,112 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T14:52:37,112 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,112 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,112 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:52:37,113 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T14:52:37,113 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,114 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:52:37,114 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:52:37,114 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T14:52:37,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:52:37,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T14:52:37,116 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T14:52:37,116 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,116 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T14:52:37,116 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,117 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,117 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,118 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,118 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,118 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T14:52:37,119 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T14:52:37,121 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:52:37,122 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881842, jitterRate=0.12132041156291962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T14:52:37,123 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733237557109Initializing all the Stores at 1733237557110 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237557110Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237557111 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237557111Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237557111Cleaning up temporary data from old regions at 1733237557118 (+7 ms)Region opened successfully at 1733237557122 (+4 ms)
2024-12-03T14:52:37,123 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T14:52:37,126 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30690e8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:52:37,127 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-03T14:52:37,127 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T14:52:37,127 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T14:52:37,127 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T14:52:37,128 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T14:52:37,128 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-03T14:52:37,129 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T14:52:37,132 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T14:52:37,133 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T14:52:37,134 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-03T14:52:37,135 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T14:52:37,135 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T14:52:37,136 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-03T14:52:37,136 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T14:52:37,137 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T14:52:37,138 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-03T14:52:37,139 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T14:52:37,139 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T14:52:37,141 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T14:52:37,142 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T14:52:37,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:52:37,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T14:52:37,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,143 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,46245,1733237556973, sessionid=0x100a091a34f0000, setting cluster-up flag (Was=false)
2024-12-03T14:52:37,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,147 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T14:52:37,148 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,152 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T14:52:37,153 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,154 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-03T14:52:37,155 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-03T14:52:37,155 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-03T14:52:37,155 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T14:52:37,156 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,46245,1733237556973
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:52:37,157 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,160 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237587160
2024-12-03T14:52:37,161 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T14:52:37,161 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T14:52:37,161 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T14:52:37,161 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T14:52:37,161 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T14:52:37,161 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:52:37,161 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T14:52:37,161 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-03T14:52:37,162 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,162 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T14:52:37,163 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,163 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T14:52:37,163 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T14:52:37,163 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T14:52:37,165 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T14:52:37,165 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T14:52:37,165 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237557165,5,FailOnTimeoutGroup]
2024-12-03T14:52:37,165 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237557165,5,FailOnTimeoutGroup]
2024-12-03T14:52:37,165 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,166 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T14:52:37,166 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,166 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:52:37,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741831_1007 (size=1321)
2024-12-03T14:52:37,171 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-03T14:52:37,171 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4
2024-12-03T14:52:37,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:52:37,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741832_1008 (size=32)
2024-12-03T14:52:37,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:52:37,182 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:52:37,183 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:52:37,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,184 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,184 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:52:37,185 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:52:37,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,185 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,185 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:52:37,186 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:52:37,186 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,187 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,187 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:52:37,188 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:52:37,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,188 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:52:37,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740
2024-12-03T14:52:37,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740
2024-12-03T14:52:37,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:52:37,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:52:37,190 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:52:37,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:52:37,193 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T14:52:37,193 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=762560, jitterRate=-0.030355125665664673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:52:37,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733237557181Initializing all the Stores at 1733237557182 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237557182Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237557182Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237557182Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237557182Cleaning up temporary data from old regions at 1733237557190 (+8 ms)Region opened successfully at 1733237557193 (+3 ms)
2024-12-03T14:52:37,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:52:37,193 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:52:37,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:52:37,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:52:37,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:52:37,197 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:52:37,197 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237557193Disabling compacts and flushes for region at 1733237557193Disabling writes for close at 1733237557193Writing region close event to WAL at 1733237557197 (+4 ms)Closed at 1733237557197
2024-12-03T14:52:37,198 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:52:37,198 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-03T14:52:37,198 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T14:52:37,199 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:52:37,200 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T14:52:37,230 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(746): ClusterId : d58174b8-6648-4218-83a0-ad29b19b1be3
2024-12-03T14:52:37,230 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T14:52:37,231 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T14:52:37,231 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T14:52:37,233 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T14:52:37,233 DEBUG [RS:0;a5d22df9eca2:45679 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ec3773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0
2024-12-03T14:52:37,244 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:45679
2024-12-03T14:52:37,245 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-03T14:52:37,245 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-03T14:52:37,245 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-03T14:52:37,245 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,46245,1733237556973 with port=45679, startcode=1733237557013
2024-12-03T14:52:37,246 DEBUG [RS:0;a5d22df9eca2:45679 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T14:52:37,247 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T14:52:37,248 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46245 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,248 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46245 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,249 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4
2024-12-03T14:52:37,249 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44885
2024-12-03T14:52:37,249 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-03T14:52:37,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:52:37,251 DEBUG [RS:0;a5d22df9eca2:45679 {}] zookeeper.ZKUtil(111): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,251 WARN  [RS:0;a5d22df9eca2:45679 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T14:52:37,251 INFO  [RS:0;a5d22df9eca2:45679 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:52:37,251 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,45679,1733237557013]
2024-12-03T14:52:37,251 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,254 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T14:52:37,255 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T14:52:37,255 INFO  [RS:0;a5d22df9eca2:45679 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T14:52:37,255 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,256 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-03T14:52:37,257 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-03T14:52:37,257 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:52:37,257 DEBUG [RS:0;a5d22df9eca2:45679 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3
2024-12-03T14:52:37,258 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,259 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,259 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,259 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,259 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,259 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,45679,1733237557013-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:52:37,270 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T14:52:37,271 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,45679,1733237557013-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,271 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,271 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.Replication(171): a5d22df9eca2,45679,1733237557013 started
2024-12-03T14:52:37,282 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,282 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,45679,1733237557013, RpcServer on a5d22df9eca2/172.17.0.2:45679, sessionid=0x100a091a34f0001
2024-12-03T14:52:37,282 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T14:52:37,282 DEBUG [RS:0;a5d22df9eca2:45679 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,282 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,45679,1733237557013'
2024-12-03T14:52:37,282 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T14:52:37,283 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T14:52:37,283 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T14:52:37,283 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T14:52:37,283 DEBUG [RS:0;a5d22df9eca2:45679 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,283 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,45679,1733237557013'
2024-12-03T14:52:37,283 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T14:52:37,284 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T14:52:37,284 DEBUG [RS:0;a5d22df9eca2:45679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T14:52:37,284 INFO  [RS:0;a5d22df9eca2:45679 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T14:52:37,284 INFO  [RS:0;a5d22df9eca2:45679 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T14:52:37,350 WARN  [a5d22df9eca2:46245 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-03T14:52:37,386 INFO  [RS:0;a5d22df9eca2:45679 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C45679%2C1733237557013, suffix=, logDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/a5d22df9eca2,45679,1733237557013, archiveDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/oldWALs, maxLogs=32
2024-12-03T14:52:37,387 INFO  [RS:0;a5d22df9eca2:45679 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C45679%2C1733237557013.1733237557387
2024-12-03T14:52:37,395 INFO  [RS:0;a5d22df9eca2:45679 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/a5d22df9eca2,45679,1733237557013/a5d22df9eca2%2C45679%2C1733237557013.1733237557387
2024-12-03T14:52:37,396 DEBUG [RS:0;a5d22df9eca2:45679 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39807:39807),(127.0.0.1/127.0.0.1:42863:42863)]
2024-12-03T14:52:37,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,44475,1733237382458/a5d22df9eca2%2C44475%2C1733237382458.1733237382649
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:37,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/WALs/a5d22df9eca2,34883,1733237381195/a5d22df9eca2%2C34883%2C1733237381195.meta.1733237382289.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:37,600 DEBUG [a5d22df9eca2:46245 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T14:52:37,602 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,605 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,45679,1733237557013, state=OPENING
2024-12-03T14:52:37,607 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T14:52:37,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,611 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T14:52:37,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:52:37,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:52:37,612 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,45679,1733237557013}]
2024-12-03T14:52:37,764 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T14:52:37,766 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T14:52:37,771 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-03T14:52:37,771 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:52:37,773 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C45679%2C1733237557013.meta, suffix=.meta, logDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/a5d22df9eca2,45679,1733237557013, archiveDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/oldWALs, maxLogs=32
2024-12-03T14:52:37,774 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a5d22df9eca2%2C45679%2C1733237557013.meta.1733237557774.meta
2024-12-03T14:52:37,779 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/a5d22df9eca2,45679,1733237557013/a5d22df9eca2%2C45679%2C1733237557013.meta.1733237557774.meta
2024-12-03T14:52:37,785 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39807:39807),(127.0.0.1/127.0.0.1:42863:42863)]
2024-12-03T14:52:37,789 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T14:52:37,790 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T14:52:37,790 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T14:52:37,790 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T14:52:37,790 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T14:52:37,790 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T14:52:37,790 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-03T14:52:37,790 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-03T14:52:37,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T14:52:37,792 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T14:52:37,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-03T14:52:37,793 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-03T14:52:37,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,794 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T14:52:37,794 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T14:52:37,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,794 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,794 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T14:52:37,795 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T14:52:37,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T14:52:37,795 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T14:52:37,795 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-03T14:52:37,796 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740
2024-12-03T14:52:37,797 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740
2024-12-03T14:52:37,798 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-03T14:52:37,798 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-03T14:52:37,798 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T14:52:37,799 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-03T14:52:37,800 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795450, jitterRate=0.011467844247817993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T14:52:37,800 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-03T14:52:37,800 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733237557790Writing region info on filesystem at 1733237557790Initializing all the Stores at 1733237557791 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237557791Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237557791Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237557791Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237557791Cleaning up temporary data from old regions at 1733237557798 (+7 ms)Running coprocessor post-open hooks at 1733237557800 (+2 ms)Region opened successfully at 1733237557800
2024-12-03T14:52:37,801 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237557764
2024-12-03T14:52:37,803 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T14:52:37,803 INFO  [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-03T14:52:37,804 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,804 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,45679,1733237557013, state=OPEN
2024-12-03T14:52:37,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:52:37,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T14:52:37,806 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:52:37,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T14:52:37,808 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T14:52:37,808 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,45679,1733237557013 in 195 msec
2024-12-03T14:52:37,810 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T14:52:37,810 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 610 msec
2024-12-03T14:52:37,811 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-03T14:52:37,811 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-03T14:52:37,812 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:52:37,812 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,45679,1733237557013, seqNum=-1]
2024-12-03T14:52:37,812 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:52:37,813 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:52:37,817 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 662 msec
2024-12-03T14:52:37,817 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237557817, completionTime=-1
2024-12-03T14:52:37,817 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T14:52:37,817 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237617819
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237677819
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46245,1733237556973-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46245,1733237556973-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46245,1733237556973-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:46245, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,819 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,821 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.780sec
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46245,1733237556973-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T14:52:37,823 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46245,1733237556973-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T14:52:37,825 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-03T14:52:37,825 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T14:52:37,825 INFO  [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46245,1733237556973-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T14:52:37,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e03366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:52:37,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,46245,-1 for getting cluster id
2024-12-03T14:52:37,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-03T14:52:37,831 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd58174b8-6648-4218-83a0-ad29b19b1be3'
2024-12-03T14:52:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-03T14:52:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d58174b8-6648-4218-83a0-ad29b19b1be3"

2024-12-03T14:52:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65934ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:52:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,46245,-1]
2024-12-03T14:52:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-03T14:52:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:37,832 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-03T14:52:37,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4abee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T14:52:37,833 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-03T14:52:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,45679,1733237557013, seqNum=-1]
2024-12-03T14:52:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T14:52:37,835 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T14:52:37,837 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,837 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T14:52:37,839 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-03T14:52:37,839 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T14:52:37,842 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/test.com,8080,1, archiveDir=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/oldWALs, maxLogs=32
2024-12-03T14:52:37,842 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733237557842
2024-12-03T14:52:37,847 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/test.com,8080,1/test.com%2C8080%2C1.1733237557842
2024-12-03T14:52:37,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39807:39807),(127.0.0.1/127.0.0.1:42863:42863)]
2024-12-03T14:52:37,849 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733237557849
2024-12-03T14:52:37,855 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,855 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,855 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,856 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,856 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,856 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/test.com,8080,1/test.com%2C8080%2C1.1733237557842 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/test.com,8080,1/test.com%2C8080%2C1.1733237557849
2024-12-03T14:52:37,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741835_1011 (size=93)
2024-12-03T14:52:37,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741835_1011 (size=93)
2024-12-03T14:52:37,861 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42863:42863),(127.0.0.1/127.0.0.1:39807:39807)]
2024-12-03T14:52:37,861 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/WALs/test.com,8080,1/test.com%2C8080%2C1.1733237557842 to hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/oldWALs/test.com%2C8080%2C1.1733237557842
2024-12-03T14:52:37,862 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,862 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,862 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,862 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,862 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:37,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741836_1012 (size=93)
2024-12-03T14:52:37,864 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741836_1012 (size=93)
2024-12-03T14:52:37,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/oldWALs
2024-12-03T14:52:37,865 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733237557849)
2024-12-03T14:52:37,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-03T14:52:37,866 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:52:37,866 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:52:37,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:37,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:37,866 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-03T14:52:37,866 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T14:52:37,866 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1907722284, stopped=false
2024-12-03T14:52:37,866 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,46245,1733237556973
2024-12-03T14:52:37,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:52:37,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T14:52:37,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:37,867 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:52:37,867 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-03T14:52:37,867 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:52:37,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:37,867 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,45679,1733237557013' *****
2024-12-03T14:52:37,867 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:52:37,868 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T14:52:37,868 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:45679.
2024-12-03T14:52:37,868 DEBUG [RS:0;a5d22df9eca2:45679 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-03T14:52:37,868 DEBUG [RS:0;a5d22df9eca2:45679 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-03T14:52:37,868 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-03T14:52:37,868 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-03T14:52:37,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T14:52:37,868 DEBUG [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-03T14:52:37,868 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-03T14:52:37,868 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-03T14:52:37,868 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-03T14:52:37,868 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T14:52:37,869 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T14:52:37,869 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-03T14:52:37,882 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740/.tmp/ns/fdb3e58e69234b4e8e485d12fb8d03b7 is 43, key is default/ns:d/1733237557813/Put/seqid=0
2024-12-03T14:52:37,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741837_1013 (size=5153)
2024-12-03T14:52:37,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741837_1013 (size=5153)
2024-12-03T14:52:37,887 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740/.tmp/ns/fdb3e58e69234b4e8e485d12fb8d03b7
2024-12-03T14:52:37,892 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740/.tmp/ns/fdb3e58e69234b4e8e485d12fb8d03b7 as hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740/ns/fdb3e58e69234b4e8e485d12fb8d03b7
2024-12-03T14:52:37,896 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740/ns/fdb3e58e69234b4e8e485d12fb8d03b7, entries=2, sequenceid=6, filesize=5.0 K
2024-12-03T14:52:37,897 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=6, compaction requested=false
2024-12-03T14:52:37,901 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-03T14:52:37,902 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T14:52:37,902 INFO  [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-03T14:52:37,902 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733237557868Running coprocessor pre-close hooks at 1733237557868Disabling compacts and flushes for region at 1733237557868Disabling writes for close at 1733237557868Obtaining lock to block concurrent updates at 1733237557869 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733237557869Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733237557869Flushing stores of hbase:meta,,1.1588230740 at 1733237557869Flushing 1588230740/ns: creating writer at 1733237557870 (+1 ms)Flushing 1588230740/ns: appending metadata at 1733237557882 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733237557882Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bb0471f: reopening flushed file at 1733237557891 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=6, compaction requested=false at 1733237557897 (+6 ms)Writing region close event to WAL at 1733237557898 (+1 ms)Running coprocessor post-close hooks at 1733237557902 (+4 ms)Closed at 1733237557902
2024-12-03T14:52:37,902 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T14:52:37,981 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35825/user/jenkins/test-data/ed5477eb-7293-b821-c47f-557a0e2aa187/MasterData/WALs/a5d22df9eca2,37049,1733237381157/a5d22df9eca2%2C37049%2C1733237381157.1733237381716
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T14:52:38,069 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,45679,1733237557013; all regions closed.
2024-12-03T14:52:38,069 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,069 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,070 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,070 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,070 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741834_1010 (size=1152)
2024-12-03T14:52:38,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741834_1010 (size=1152)
2024-12-03T14:52:38,076 DEBUG [RS:0;a5d22df9eca2:45679 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/oldWALs
2024-12-03T14:52:38,076 INFO  [RS:0;a5d22df9eca2:45679 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C45679%2C1733237557013.meta:.meta(num 1733237557774)
2024-12-03T14:52:38,077 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,077 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,077 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,077 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,077 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741833_1009 (size=93)
2024-12-03T14:52:38,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741833_1009 (size=93)
2024-12-03T14:52:38,084 DEBUG [RS:0;a5d22df9eca2:45679 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/oldWALs
2024-12-03T14:52:38,084 INFO  [RS:0;a5d22df9eca2:45679 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a5d22df9eca2%2C45679%2C1733237557013:(num 1733237557387)
2024-12-03T14:52:38,084 DEBUG [RS:0;a5d22df9eca2:45679 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T14:52:38,084 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T14:52:38,084 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:52:38,085 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-03T14:52:38,085 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:52:38,085 INFO  [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:52:38,085 INFO  [RS:0;a5d22df9eca2:45679 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45679
2024-12-03T14:52:38,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T14:52:38,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,45679,1733237557013
2024-12-03T14:52:38,087 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:52:38,088 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,45679,1733237557013]
2024-12-03T14:52:38,089 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,45679,1733237557013 already deleted, retry=false
2024-12-03T14:52:38,089 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,45679,1733237557013 expired; onlineServers=0
2024-12-03T14:52:38,089 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,46245,1733237556973' *****
2024-12-03T14:52:38,089 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T14:52:38,089 INFO  [M:0;a5d22df9eca2:46245 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-03T14:52:38,089 INFO  [M:0;a5d22df9eca2:46245 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-03T14:52:38,089 DEBUG [M:0;a5d22df9eca2:46245 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T14:52:38,089 DEBUG [M:0;a5d22df9eca2:46245 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T14:52:38,089 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T14:52:38,089 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237557165 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237557165,5,FailOnTimeoutGroup]
2024-12-03T14:52:38,089 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237557165 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237557165,5,FailOnTimeoutGroup]
2024-12-03T14:52:38,089 INFO  [M:0;a5d22df9eca2:46245 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-03T14:52:38,090 INFO  [M:0;a5d22df9eca2:46245 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-03T14:52:38,090 DEBUG [M:0;a5d22df9eca2:46245 {}] master.HMaster(1795): Stopping service threads
2024-12-03T14:52:38,090 INFO  [M:0;a5d22df9eca2:46245 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T14:52:38,090 INFO  [M:0;a5d22df9eca2:46245 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-03T14:52:38,090 INFO  [M:0;a5d22df9eca2:46245 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T14:52:38,090 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T14:52:38,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T14:52:38,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T14:52:38,091 DEBUG [M:0;a5d22df9eca2:46245 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-12-03T14:52:38,092 DEBUG [M:0;a5d22df9eca2:46245 {}] master.ActiveMasterManager(353): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-12-03T14:52:38,092 INFO  [M:0;a5d22df9eca2:46245 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/.lastflushedseqids
2024-12-03T14:52:38,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741838_1014 (size=108)
2024-12-03T14:52:38,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741838_1014 (size=108)
2024-12-03T14:52:38,100 INFO  [M:0;a5d22df9eca2:46245 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-03T14:52:38,100 INFO  [M:0;a5d22df9eca2:46245 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T14:52:38,100 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T14:52:38,100 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:38,100 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:38,100 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T14:52:38,100 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:38,100 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-03T14:52:38,120 DEBUG [M:0;a5d22df9eca2:46245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa09647d66424978904be7b0c24201f7 is 82, key is hbase:meta,,1/info:regioninfo/1733237557803/Put/seqid=0
2024-12-03T14:52:38,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741839_1015 (size=5672)
2024-12-03T14:52:38,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741839_1015 (size=5672)
2024-12-03T14:52:38,125 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa09647d66424978904be7b0c24201f7
2024-12-03T14:52:38,141 DEBUG [M:0;a5d22df9eca2:46245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7639d886363343b88a35db4d032d037d is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733237557816/Put/seqid=0
2024-12-03T14:52:38,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741840_1016 (size=5275)
2024-12-03T14:52:38,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741840_1016 (size=5275)
2024-12-03T14:52:38,146 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7639d886363343b88a35db4d032d037d
2024-12-03T14:52:38,163 DEBUG [M:0;a5d22df9eca2:46245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/61d986045c3a4ab6a58dadcc90aab2c8 is 69, key is a5d22df9eca2,45679,1733237557013/rs:state/1733237557248/Put/seqid=0
2024-12-03T14:52:38,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741841_1017 (size=5156)
2024-12-03T14:52:38,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741841_1017 (size=5156)
2024-12-03T14:52:38,169 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/61d986045c3a4ab6a58dadcc90aab2c8
2024-12-03T14:52:38,184 DEBUG [M:0;a5d22df9eca2:46245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/38bbf9fdd725417eabde34f293c2e6d0 is 52, key is load_balancer_on/state:d/1733237557838/Put/seqid=0
2024-12-03T14:52:38,188 INFO  [RS:0;a5d22df9eca2:45679 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:52:38,188 INFO  [RS:0;a5d22df9eca2:45679 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,45679,1733237557013; zookeeper connection closed.
2024-12-03T14:52:38,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:38,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741842_1018 (size=5056)
2024-12-03T14:52:38,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45679-0x100a091a34f0001, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:38,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741842_1018 (size=5056)
2024-12-03T14:52:38,188 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@481f51ee {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@481f51ee
2024-12-03T14:52:38,189 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T14:52:38,189 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/38bbf9fdd725417eabde34f293c2e6d0
2024-12-03T14:52:38,193 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa09647d66424978904be7b0c24201f7 as hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa09647d66424978904be7b0c24201f7
2024-12-03T14:52:38,196 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa09647d66424978904be7b0c24201f7, entries=8, sequenceid=29, filesize=5.5 K
2024-12-03T14:52:38,197 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7639d886363343b88a35db4d032d037d as hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7639d886363343b88a35db4d032d037d
2024-12-03T14:52:38,201 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7639d886363343b88a35db4d032d037d, entries=3, sequenceid=29, filesize=5.2 K
2024-12-03T14:52:38,202 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/61d986045c3a4ab6a58dadcc90aab2c8 as hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/61d986045c3a4ab6a58dadcc90aab2c8
2024-12-03T14:52:38,206 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/61d986045c3a4ab6a58dadcc90aab2c8, entries=1, sequenceid=29, filesize=5.0 K
2024-12-03T14:52:38,207 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/38bbf9fdd725417eabde34f293c2e6d0 as hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/38bbf9fdd725417eabde34f293c2e6d0
2024-12-03T14:52:38,211 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/77fea8e4-e1dd-ed2d-4ec7-33a6d49558c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/38bbf9fdd725417eabde34f293c2e6d0, entries=1, sequenceid=29, filesize=4.9 K
2024-12-03T14:52:38,212 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=29, compaction requested=false
2024-12-03T14:52:38,214 INFO  [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T14:52:38,214 DEBUG [M:0;a5d22df9eca2:46245 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733237558100Disabling compacts and flushes for region at 1733237558100Disabling writes for close at 1733237558100Obtaining lock to block concurrent updates at 1733237558100Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237558100Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733237558101 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237558101Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237558101Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237558119 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237558119Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237558129 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237558141 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237558141Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237558149 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237558162 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237558162Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733237558172 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733237558183 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733237558183Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e742b03: reopening flushed file at 1733237558192 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f9619ef: reopening flushed file at 1733237558196 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23a9b293: reopening flushed file at 1733237558201 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3304e8c4: reopening flushed file at 1733237558206 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=29, compaction requested=false at 1733237558212 (+6 ms)Writing region close event to WAL at 1733237558214 (+2 ms)Closed at 1733237558214
2024-12-03T14:52:38,214 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,214 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,214 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,214 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,214 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-03T14:52:38,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37441 is added to blk_1073741830_1006 (size=10311)
2024-12-03T14:52:38,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46219 is added to blk_1073741830_1006 (size=10311)
2024-12-03T14:52:38,217 INFO  [M:0;a5d22df9eca2:46245 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-03T14:52:38,217 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-03T14:52:38,217 INFO  [M:0;a5d22df9eca2:46245 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46245
2024-12-03T14:52:38,217 INFO  [M:0;a5d22df9eca2:46245 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-03T14:52:38,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:38,318 INFO  [M:0;a5d22df9eca2:46245 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-03T14:52:38,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46245-0x100a091a34f0000, quorum=127.0.0.1:63628, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T14:52:38,322 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@380ffe40{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:52:38,323 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e35321a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:52:38,323 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:52:38,323 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1926aa54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:52:38,324 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b825638{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.log.dir/,STOPPED}
2024-12-03T14:52:38,326 WARN  [BP-1438476385-172.17.0.2-1733237556415 heartbeating to localhost/127.0.0.1:44885 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:52:38,326 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:52:38,326 WARN  [BP-1438476385-172.17.0.2-1733237556415 heartbeating to localhost/127.0.0.1:44885 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1438476385-172.17.0.2-1733237556415 (Datanode Uuid daa912b9-4b4f-456d-86ca-5703ba74c39f) service to localhost/127.0.0.1:44885
2024-12-03T14:52:38,326 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:52:38,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data3/current/BP-1438476385-172.17.0.2-1733237556415 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:38,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data4/current/BP-1438476385-172.17.0.2-1733237556415 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:38,327 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:52:38,330 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@150dab73{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T14:52:38,330 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6486a7e6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:52:38,330 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:52:38,330 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10c1adfc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:52:38,330 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5be97557{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.log.dir/,STOPPED}
2024-12-03T14:52:38,331 WARN  [BP-1438476385-172.17.0.2-1733237556415 heartbeating to localhost/127.0.0.1:44885 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T14:52:38,331 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T14:52:38,331 WARN  [BP-1438476385-172.17.0.2-1733237556415 heartbeating to localhost/127.0.0.1:44885 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1438476385-172.17.0.2-1733237556415 (Datanode Uuid 3c2adf39-8364-4393-aadb-950429fbcd1b) service to localhost/127.0.0.1:44885
2024-12-03T14:52:38,331 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T14:52:38,332 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data1/current/BP-1438476385-172.17.0.2-1733237556415 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:38,332 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/cluster_63eb1520-ba81-0b0b-a5c3-eb2cfba9ed84/data/data2/current/BP-1438476385-172.17.0.2-1733237556415 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T14:52:38,332 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T14:52:38,337 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4bedee20{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T14:52:38,338 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@27862096{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T14:52:38,338 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T14:52:38,338 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@412c5a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T14:52:38,338 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fee3652{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/375a822d-f0bc-7fa1-a941-b2d0a051366f/hadoop.log.dir/,STOPPED}
2024-12-03T14:52:38,344 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-03T14:52:38,359 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-03T14:52:38,369 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=271 (was 231)
Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44885
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44885 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44885 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44885
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:44885
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44885
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: globalEventExecutor-1-19
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113)
	app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44885
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44885 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 143) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4663 (was 4660) - AvailableMemoryMB LEAK? -